Skip to content

Commit

Permalink
Sept updates (#69)
Browse files Browse the repository at this point in the history
* Moving default version to 2.6.1

* Default k8s on for ingress

* Proxy config to use brokerSts for discovery

* Use ClusterIP to access ZooKeeper for broker and bookie

* Move zoonaviagator

* Move zookeeper files

* Only create priority class if rbac is enabled

* Add non-persistent zookeeper option; new bookkeeper health check; option to disable wait containers; alll components use zookeeper ClusterIP address; examples for configuring resource limits; option for extra proxy service with different listen ports
  • Loading branch information
cdbartholomew authored Oct 25, 2020
1 parent 49514b2 commit 6a346f1
Show file tree
Hide file tree
Showing 27 changed files with 660 additions and 27 deletions.
14 changes: 14 additions & 0 deletions helm-chart-sources/pulsar/templates/_helpers.tpl
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,8 @@ Create chart name and version as used by the chart label.
{{- define "pulsar.zkConnectString" -}}
{{- $global := . -}}
{{- range $i, $e := until (.Values.zookeeper.replicaCount | int) -}}{{ if ne $i 0 }},{{ end }}{{ template "pulsar.fullname" $global }}-{{ $global.Values.zookeeper.component }}-{{ printf "%d" $i }}.{{ template "pulsar.fullname" $global }}-{{ $global.Values.zookeeper.component }}{{ end }}
{{- $global := . -}}
{{- range $i, $e := until (.Values.zookeepernp.replicaCount | int) -}},{{ template "pulsar.fullname" $global }}-{{ $global.Values.zookeepernp.component }}-{{ printf "%d" $i }}.{{ template "pulsar.fullname" $global }}-{{ $global.Values.zookeepernp.component }}{{ end }}
{{- end -}}

{{- define "pulsar.bkConnectString" -}}
Expand All @@ -60,6 +62,9 @@ Create chart name and version as used by the chart label.
{{- $global := . -}}
{{- $port := "2281" -}}
{{- range $i, $e := until (.Values.zookeeper.replicaCount | int) -}}{{ if ne $i 0 }},{{ end }}{{ template "pulsar.fullname" $global }}-{{ $global.Values.zookeeper.component }}-{{ printf "%d" $i }}.{{ template "pulsar.fullname" $global }}-{{ $global.Values.zookeeper.component }}:{{ $port }}{{ end }}
{{- $global := . -}}
{{- $port := "2281" -}}
{{- range $i, $e := until (.Values.zookeepernp.replicaCount | int) -}},{{ template "pulsar.fullname" $global }}-{{ $global.Values.zookeepernp.component }}-{{ printf "%d" $i }}.{{ template "pulsar.fullname" $global }}-{{ $global.Values.zookeepernp.component }}:{{ $port }}{{ end }}
{{- end -}}

{{- define "pulsar.zkConnectStringTlsONe" -}}
Expand All @@ -71,4 +76,13 @@ Create chart name and version as used by the chart label.
{{- define "pulsar.zkServers" -}}
{{- $global := . }}
{{- range $i, $e := until (.Values.zookeeper.replicaCount | int) -}}{{ if ne $i 0 }},{{ end }}{{ template "pulsar.fullname" $global }}-{{ $global.Values.zookeeper.component }}-{{ printf "%d" $i }}{{ end }}
{{- $global := . }}
{{- range $i, $e := until (.Values.zookeepernp.replicaCount | int) -}},{{ template "pulsar.fullname" $global }}-{{ $global.Values.zookeepernp.component }}-{{ printf "%d" $i }}{{ end }}
{{- end -}}

{{- define "pulsar.zkDomains" -}}
{{- $global := . -}}
{{- range $i, $e := until (.Values.zookeeper.replicaCount | int) -}}{{ if ne $i 0 }},{{ end }}{{ template "pulsar.fullname" $global }}-{{ $global.Values.zookeeper.component }}-{{ printf "%d" $i }}.{{ template "pulsar.fullname" $global }}-{{ $global.Values.zookeeper.component }}{{ end }}
{{- $global := . -}}
{{- range $i, $e := until (.Values.zookeepernp.replicaCount | int) -}},{{ template "pulsar.fullname" $global }}-{{ $global.Values.zookeepernp.component }}-{{ printf "%d" $i }}.{{ template "pulsar.fullname" $global }}-{{ $global.Values.zookeepernp.component }}{{ end }}
{{- end -}}
12 changes: 10 additions & 2 deletions helm-chart-sources/pulsar/templates/autorecovery-configmap.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -33,9 +33,17 @@ metadata:
data:
zkServers:
{{- if and .Values.enableTls .Values.tls.zookeeper.enabled }}
{{ include "pulsar.zkConnectStringTls" . }}
{{- if .Values.extra.zookeepernp }}
{{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.component }}-ca:2281,{{ template "pulsar.fullname" . }}-{{ .Values.zookeepernp.component }}-ca:2281
{{- else }}
{{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.component }}-ca:2281
{{- end }}
{{- else }}
{{ include "pulsar.zkConnectString" . }}
{{- if .Values.extra.zookeepernp }}
{{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.component }}-ca:2181,{{ template "pulsar.fullname" . }}-{{ .Values.zookeepernp.component }}-ca:2181
{{- else }}
{{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.component }}-ca:2181
{{- end }}
{{- end }}
{{ toYaml .Values.autoRecovery.configData | indent 2 }}
{{- end }}
10 changes: 10 additions & 0 deletions helm-chart-sources/pulsar/templates/bookkeeper-configmap.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -32,14 +32,24 @@ metadata:
data:
zkServers:
{{- if and .Values.enableTls .Values.tls.zookeeper.enabled }}
{{- if .Values.extra.zookeepernp }}
{{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.component }}-ca:2281,{{ template "pulsar.fullname" . }}-{{ .Values.zookeepernp.component }}-ca:2281
{{- else }}
{{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.component }}-ca:2281
{{- end }}
{{- else }}
{{- if .Values.extra.zookeepernp }}
{{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.component }}-ca:2181,{{ template "pulsar.fullname" . }}-{{ .Values.zookeepernp.component }}-ca:2181
{{- else }}
{{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.component }}-ca:2181
{{- end }}
{{- end }}
# disable auto recovery on bookies since we will start AutoRecovery in separated pods
autoRecoveryDaemonEnabled: "false"
# In k8s always want to use hostname as bookie ID since IP addresses are ephemeral
useHostNameAsBookieID: "true"
# HTTP server used by health check
httpServerEnabled: "true"
{{- if .Values.function.enableStateStorage }}
{{- if not (or .Values.extra.stateStorage .Values.function.stateStorageUrlOverride) }}
# Enable function state storage
Expand Down
11 changes: 9 additions & 2 deletions helm-chart-sources/pulsar/templates/bookkeeper-statefulset.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -148,8 +148,15 @@ spec:
imagePullPolicy: {{ .Values.image.bookkeeper.pullPolicy }}
{{- if .Values.bookkeeper.probe.enabled }}
readinessProbe:
tcpSocket:
port: {{ .Values.bookkeeper.probe.port }}
httpGet:
path: /api/v1/bookie/is_ready
port: 8000
initialDelaySeconds: {{ .Values.bookkeeper.probe.initial }}
periodSeconds: {{ .Values.bookkeeper.probe.period }}
livenessProbe:
httpGet:
path: /api/v1/bookie/is_ready
port: 8000
initialDelaySeconds: {{ .Values.bookkeeper.probe.initial }}
periodSeconds: {{ .Values.bookkeeper.probe.period }}
{{- end }}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,16 +32,32 @@ metadata:
data:
zookeeperServers:
{{- if and .Values.enableTls .Values.tls.zookeeper.enabled }}
{{- if .Values.extra.zookeepernp }}
{{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.component }}-ca:2281,{{ template "pulsar.fullname" . }}-{{ .Values.zookeepernp.component }}-ca:2281
{{- else }}
{{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.component }}-ca:2281
{{- end }}
{{- else }}
{{- if .Values.extra.zookeepernp }}
{{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.component }}-ca:2181,{{ template "pulsar.fullname" . }}-{{ .Values.zookeepernp.component }}-ca:2181
{{- else }}
{{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.component }}-ca:2181
{{- end }}
{{- end }}
configurationStoreServers:
{{- if and .Values.enableTls .Values.tls.zookeeper.enabled }}
{{- if .Values.extra.zookeepernp }}
{{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.component }}-ca:2281,{{ template "pulsar.fullname" . }}-{{ .Values.zookeepernp.component }}-ca:2281
{{- else }}
{{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.component }}-ca:2281
{{- end }}
{{- else }}
{{- if .Values.extra.zookeepernp }}
{{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.component }}-ca:2181,{{ template "pulsar.fullname" . }}-{{ .Values.zookeepernp.component }}-ca:2181
{{- else }}
{{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.component }}-ca:2181
{{- end }}
{{- end }}
clusterName: {{ template "pulsar.fullname" . }}
{{- if .Values.broker.functionsWorkerEnabled }}
functionsWorkerEnabled: "true"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -175,6 +175,7 @@ spec:
emptyDir: {}
{{- end }}
initContainers:
{{- if .Values.enableWaitContainers }}
# This init container will wait for bookkeeper to be ready before
# deploying the broker
- name: wait-bookkeeper-ready
Expand All @@ -186,6 +187,7 @@ spec:
until nslookup {{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}-{{ add (.Values.bookkeeper.replicaCount | int) -1 }}.{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}.{{ .Release.Namespace }}; do
sleep 3;
done;
{{- end }}
{{- if .Values.broker.initContainer }}
- name: add-libs
image: "{{ .Values.broker.initContainer.repository }}:{{ .Values.broker.initContainer.tag }}"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,16 +32,32 @@ metadata:
data:
zookeeperServers:
{{- if and .Values.enableTls .Values.tls.zookeeper.enabled }}
{{- if .Values.extra.zookeepernp }}
{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.component }}-ca:2281,{{ template "pulsar.fullname" . }}-{{ .Values.zookeepernp.component }}-ca:2281
{{- else }}
{{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.component }}-ca:2281
{{- end }}
{{- else }}
{{- if .Values.extra.zookeepernp }}
{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.component }}-ca:2181,{{ template "pulsar.fullname" . }}-{{ .Values.zookeepernp.component }}-ca:2181
{{- else }}
{{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.component }}-ca:2181
{{- end }}
{{- end }}
configurationStoreServers:
{{- if and .Values.enableTls .Values.tls.zookeeper.enabled }}
{{- if .Values.extra.zookeepernp }}
{{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.component }}-ca:2281,{{ template "pulsar.fullname" . }}-{{ .Values.zookeepernp.component }}-ca:2281
{{- else }}
{{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.component }}-ca:2281
{{- end }}
{{- else }}
{{- if .Values.extra.zookeepernp }}
{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.component }}-ca:2181,{{ template "pulsar.fullname" . }}-{{ .Values.zookeepernp.component }}-ca:2181
{{- else }}
{{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.component }}-ca:2181
{{- end }}
{{- end }}
clusterName: {{ template "pulsar.fullname" . }}
{{- if .Values.brokerSts.functionsWorkerEnabled }}
functionsWorkerEnabled: "true"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -172,6 +172,7 @@ spec:
emptyDir: {}
{{- end }}
initContainers:
{{- if .Values.enableWaitContainers }}
# This init container will wait for bookkeeper to be ready before
# deploying the broker
- name: wait-bookkeeper-ready
Expand All @@ -183,6 +184,7 @@ spec:
until nslookup {{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}-{{ add (.Values.bookkeeper.replicaCount | int) -1 }}.{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}.{{ .Release.Namespace }}; do
sleep 3;
done;
{{- end }}
{{- if .Values.brokerSts.initContainer }}
- name: add-libs
image: "{{ .Values.brokerSts.initContainer.repository }}:{{ .Values.brokerSts.initContainer.tag }}"
Expand Down
34 changes: 28 additions & 6 deletions helm-chart-sources/pulsar/templates/function-configmap.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -32,14 +32,30 @@ metadata:
data:
functions_worker.yml: |-
{{- if and .Values.enableTls .Values.tls.zookeeper.enabled }}
configurationStoreServers: {{ include "pulsar.zkConnectStringTls" . }}
{{- if .Values.extra.zookeepernp }}
configurationStoreServers: {{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.component }}-ca:2281,{{ template "pulsar.fullname" . }}-{{ .Values.zookeepernp.component }}-ca:2281
{{- else }}
configurationStoreServers: {{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.component }}-ca:2281
{{- end }}
{{- else }}
configurationStoreServers: {{ include "pulsar.zkConnectString" . }}
{{- if .Values.extra.zookeepernp }}
configurationStoreServers: {{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.component }}-ca:2181,{{ template "pulsar.fullname" . }}-{{ .Values.zookeepernp.component }}-ca:2181
{{- else }}
configurationStoreServers: {{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.component }}-ca:2181
{{- end }}
{{- end }}
{{- if and .Values.enableTls .Values.tls.zookeeper.enabled }}
zookeeperServers: {{ include "pulsar.zkConnectStringTls" . }}
{{- if .Values.extra.zookeepernp }}
zookeeperServers: {{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.component }}-ca:2281,{{ template "pulsar.fullname" . }}-{{ .Values.zookeepernp.component }}-ca:2281
{{- else }}
zookeeperServers: {{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.component }}-ca:2281
{{- end }}
{{- else }}
zookeeperServers: {{ include "pulsar.zkConnectString" . }}
{{- if .Values.extra.zookeepernp }}
zookeeperServers: {{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.component }}-ca:2181,{{ template "pulsar.fullname" . }}-{{ .Values.zookeepernp.component }}-ca:2181
{{- else }}
zookeeperServers: {{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.component }}-ca:2181
{{- end }}
{{- end }}
zooKeeperSessionTimeoutMillis: 30000
pulsarFunctionsCluster: "{{ template "pulsar.fullname" . }}"
Expand Down Expand Up @@ -100,7 +116,7 @@ data:
properties:
tokenPublicKey: "file:///pulsar/token-public-key/{{ .Values.tokenPublicKeyFile }}"
clientAuthenticationPlugin: "org.apache.pulsar.client.impl.auth.AuthenticationToken"
clientAuthenticationParameters: "file:///pulsar/token-superuser/superuser.jwt"
clientAuthenticationParameters: "file:///pulsar/token-superuser/superuser.jwt"
superUserRoles:
{{- range splitList "," .Values.superUserRoles }}
- {{ . | quote }}
Expand All @@ -118,13 +134,14 @@ data:
extraFunctionDependenciesDir:
{{- end }}
{{- if eq .Values.function.runtime "kubernetes" }}
secretsProviderConfiguratorClassName: org.apache.pulsar.functions.secretsproviderconfigurator.KubernetesSecretsProviderConfigurator
kubernetesContainerFactory:
# uri to kubernetes cluster, leave it to empty and it will use the kubernetes settings in function worker
k8Uri:
# the kubernetes namespace to run the function instances. it is `default`, if this setting is left to be empty
jobNamespace: {{ .Release.Namespace }}
# the docker image to run function instance. by default it is `apachepulsar/pulsar`
pulsarDockerImageName: apachepulsar/pulsar
pulsarDockerImageName: {{ .Values.image.function.repository }}:{{ .Values.image.function.tag }}
# the root directory of pulsar home directory in `pulsarDockerImageName`. by default it is `/pulsar`.
# if you are using your own built image in `pulsarDockerImageName`, you need to set this setting accordingly
pulsarRootDir: /pulsar
Expand All @@ -145,5 +162,10 @@ data:
#extraFunctionDependenciesDir:
# Additional memory padding added on top of the memory requested by the function per on a per instance basis
percentMemoryPadding: 10
# runtimeCustomizerClassName: "org.apache.pulsar.functions.runtime.kubernetes.BasicKubernetesManifestCustomizer"
functionInstanceMinResources:
cpu: {{ .Values.function.k8sMinResources.cpu }}
ram: {{ .Values.function.k8sMinResources.ram }}
disk: {{ .Values.function.k8sMinResources.disk }}
{{- end }}
{{- end }}
26 changes: 21 additions & 5 deletions helm-chart-sources/pulsar/templates/proxy-configmap.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -44,17 +44,33 @@ data:
brokerWebServiceURLTLS: "https://{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}:8443"
{{- end }}
{{- end }}
zookeeperServers:
zookeeperServers:
{{- if and .Values.enableTls .Values.tls.zookeeper.enabled }}
"{{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.component }}-ca:2281"
{{- if .Values.extra.zookeepernp }}
{{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.component }}-ca:2281,{{ template "pulsar.fullname" . }}-{{ .Values.zookeepernp.component }}-ca:2281
{{- else }}
"{{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.component }}-ca:2181"
{{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.component }}-ca:2281
{{- end }}
{{- else }}
{{- if .Values.extra.zookeepernp }}
{{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.component }}-ca:2181,{{ template "pulsar.fullname" . }}-{{ .Values.zookeepernp.component }}-ca:2181
{{- else }}
{{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.component }}-ca:2181
{{- end }}
{{- end }}
configurationStoreServers:
{{- if and .Values.enableTls .Values.tls.zookeeper.enabled }}
"{{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.component }}-ca:2281"
{{- if .Values.extra.zookeepernp }}
{{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.component }}-ca:2281,{{ template "pulsar.fullname" . }}-{{ .Values.zookeepernp.component }}-ca:2281
{{- else }}
"{{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.component }}-ca:2181"
{{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.component }}-ca:2281
{{- end }}
{{- else }}
{{- if .Values.extra.zookeepernp }}
{{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.component }}-ca:2181,{{ template "pulsar.fullname" . }}-{{ .Values.zookeepernp.component }}-ca:2181
{{- else }}
{{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.component }}-ca:2181
{{- end }}
{{- end }}
{{- if .Values.extra.function }}
functionWorkerWebServiceURL: "http://{{ template "pulsar.fullname" . }}-{{ .Values.function.component }}-ca:6750"
Expand Down
2 changes: 2 additions & 0 deletions helm-chart-sources/pulsar/templates/proxy-deployment.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -178,6 +178,7 @@ spec:
emptyDir: {}
{{- end }}
initContainers:
{{- if .Values.enableWaitContainers }}
# This init container will wait for bookkeeper to be ready before
# deploying the booproxykies
- name: wait-bookkeeper-ready
Expand All @@ -189,6 +190,7 @@ spec:
until nslookup {{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}-{{ add (.Values.bookkeeper.replicaCount | int) -1 }}.{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}.{{ .Release.Namespace }}; do
sleep 3;
done;
{{- end }}
{{- if .Values.proxy.initContainer }}
- name: add-libs
image: "{{ .Values.proxy.initContainer.repository }}:{{ .Values.proxy.initContainer.tag }}"
Expand Down
30 changes: 30 additions & 0 deletions helm-chart-sources/pulsar/templates/proxy-service.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -45,4 +45,34 @@ spec:
app: {{ template "pulsar.name" . }}
release: {{ .Release.Name }}
component: {{ .Values.proxy.component }}
---
{{- if .Values.proxy.extraService.enabled }}
apiVersion: v1
kind: Service
metadata:
name: "{{ template "pulsar.fullname" . }}-{{ .Values.proxy.component }}-extra"
namespace: {{ .Release.Namespace }}
labels:
app: {{ template "pulsar.name" . }}
chart: {{ template "pulsar.chart" . }}
release: {{ .Release.Name }}
heritage: {{ .Release.Service }}
component: {{ .Values.proxy.component }}
cluster: {{ template "pulsar.fullname" . }}
annotations:
{{- if .Values.proxy.extraService.annotations }}
{{ toYaml .Values.proxy.extraService.annotations | indent 4 }}
{{- end }}
{{- if .Values.extra.dnsOnProxy }}
external-dns.alpha.kubernetes.io/hostname: {{ .Values.dnsName }}
{{- end }}
spec:
type: {{ .Values.proxy.extraService.type }}
ports:
{{ toYaml .Values.proxy.extraService.ports | indent 2 }}
selector:
app: {{ template "pulsar.name" . }}
release: {{ .Release.Name }}
component: {{ .Values.proxy.component }}
{{- end }}
{{- end }}
Loading

0 comments on commit 6a346f1

Please sign in to comment.