openshift-template.yaml

apiVersion: v1
kind: Template
metadata:
  name: bananas-multiservice
  annotations:
    openshift.io/display-name: "Kafka Stateful Persistent Multiservice"
    description: >-
      This template installes Apache Zookeeper and Apache Kafka clusters. For more information
      about using this template see https://github.com/strimzi/strimzi
    tags: "messaging,datastore"
    iconClass: "fa pficon-topology"
    template.openshift.io/documentation-url: "https://github.com/strimzi/strimzi"
message: "Use 'kafka:9092' as bootstrap server in your application"
parameters:
- description: Number of Zookeper cluster nodes which will be deployed (odd number of nodes is recomended)
  displayName: Number of Zookeper cluster nodes (odd number of nodes is recomended)
  name: ZOOKEEPER_NODE_COUNT
  required: true
  value: "3"
- description: Number of Kafka cluster nodes which will be deployed
  displayName: Number of Kafka cluster nodes
  name: KAFKA_NODE_COUNT
  required: true
  value: "3"
- description: Volume space available for Zookeeper data, e.g. 512Mi, 2Gi.
  displayName: Zookeeper Volume Capacity
  name: ZOOKEEPER_VOLUME_CAPACITY
  required: true
  value: 1Gi
- description: Volume space available for Kafka data, e.g. 512Mi, 2Gi.
  displayName: Kafka Volume Capacity
  name: KAFKA_VOLUME_CAPACITY
  required: true
  value: 1Gi
- description: Image repository name
  displayName: Repository Name
  name: IMAGE_REPO_NAME
  value: strimzi 
- description: Kafka image name
  displayName: Kafka image Name
  name: KAFKA_IMAGE_NAME
  value: kafka-statefulsets
- description: Kafka image tag
  displayName: Kafka image tag
  name: KAFKA_IMAGE_TAG
  value: latest
- description: Zookeeper image name
  displayName: Zookeeper image Name
  name: ZOOKEEPER_IMAGE_NAME
  value: zookeeper
- description: Zookeeper image tag
  displayName: Zookeeper image tag
  name: ZOOKEEPER_IMAGE_TAG
  value: latest
- description: Number of seconds after the container has started before healthcheck probes are initiated.
  displayName: Zookeeper healthcheck initial delay
  name: ZOOKEEPER_HEALTHCHECK_DELAY
  value: "15"
- description: Number of seconds after which the probe times out.
  displayName: Zookeeper healthcheck timeout
  name: ZOOKEEPER_HEALTHCHECK_TIMEOUT
  value: "5"
- description: Number of seconds after the container has started before healthcheck probes are initiated.
  displayName: Kafka healthcheck initial delay
  name: KAFKA_HEALTHCHECK_DELAY
  value: "15"
- description: Number of seconds after which the probe times out.
  displayName: Kafka healthcheck timeout
  name: KAFKA_HEALTHCHECK_TIMEOUT
  value: "5"
- description: Default replication factor for newly created topics
  displayName: Default replication factor
  name: KAFKA_DEFAULT_REPLICATION_FACTOR
  value: "1"
- description: Replication factor for offsets topic
  displayName: Offsets replication factor
  name: KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR
  value: "3"
- description: Replication factor for transactions state log topic
  displayName: Transaction state replication factor
  name: KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR
  value: "3"
- description: true to enable metric collection from Kafka
  displayName: Kafka metrics enabled
  name: KAFKA_METRICS_ENABLED
  value: "true"
- description: true to enable metric collection from Zookeeper
  displayName: Zookeeper metrics enabled
  name: ZOOKEEPER_METRICS_ENABLED
  value: "true"
objects:
- apiVersion: v1
  kind: Service
  metadata:
    name: kafka-${KAFKA_IMAGE_NAME}
  spec:
    ports:
    - name: kafka
      port: 9092
      targetPort: 9092
      protocol: TCP
    selector:
      name: kafka
    type: ClusterIP
- apiVersion: v1
  kind: Service
  metadata:
    name: kafka-headless-${KAFKA_IMAGE_NAME}
  spec:
    ports:
    - name: kafka
      port: 9092
      targetPort: 9092
      protocol: TCP
    selector:
      name: kafka
    type: ClusterIP
    clusterIP: None
- apiVersion: v1
  kind: Service
  metadata:
    name: zookeeper-${ZOOKEEPER_IMAGE_NAME}
  spec:
    ports:
    - name: clientport
      port: 2181
      targetPort: 2181
      protocol: TCP
    selector:
      name: zookeeper
    type: ClusterIP
- apiVersion: v1
  kind: Service
  metadata:
    name: zookeeper-headless-${ZOOKEEPER_IMAGE_NAME}
  spec:
    ports:
    - name: clientport
      port: 2181
      targetPort: 2181
      protocol: TCP
    - name: clustering
      port: 2888
      targetPort: 2888
      protocol: TCP
    - name: leaderelection
      port: 3888
      targetPort: 3888
      protocol: TCP
    selector:
      name: zookeeper
    type: ClusterIP
    clusterIP: None
- apiVersion: v1
  kind: ConfigMap
  metadata:
    name: kafka-metrics-config-${KAFKA_IMAGE_NAME}
  data:
    config.yml: |
      lowercaseOutputName: true
      rules:
      - pattern: kafka.server<type=(.+), name=(.+)PerSec\w*><>Count
        name: kafka_server_$1_$2_total
      - pattern : kafka.server<type=(.+), name=(.+)PerSec\w*, topic=(.+)><>Count
        name: kafka_server_$1_$2_total
        labels:
          topic: "$3"
- apiVersion: apps/v1beta1
  kind: StatefulSet
  metadata:
    labels:
      name: kafka-${KAFKA_IMAGE_NAME}
    name: kafka-${KAFKA_IMAGE_NAME}
  spec:
    replicas: ${KAFKA_NODE_COUNT}
    selector:
      matchLabels:
        name: kafka-${KAFKA_IMAGE_NAME}
    serviceName: kafka-headless-${ZOOKEEPER_IMAGE_NAME}
    template:
      metadata:
        labels:
          name: kafka-${KAFKA_IMAGE_NAME}
        annotations:
          prometheus.io/scrape: "true"
          prometheus.io/path: "/metrics"
          prometheus.io/port: "9404"
      spec:
        containers:
        - name: kafka-${KAFKA_IMAGE_NAME}
          image: ${IMAGE_REPO_NAME}/${KAFKA_IMAGE_NAME}:${KAFKA_IMAGE_TAG}
          imagePullPolicy: Always
          ports:
          - name: kafka
            containerPort: 9092
            protocol: TCP
          - name: kafkametrics
            containerPort: 9404
            protocol: TCP
          env:
          - name: KAFKA_DEFAULT_REPLICATION_FACTOR
            value: "${KAFKA_DEFAULT_REPLICATION_FACTOR}"
          - name: KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR
            value: "${KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR}"
          - name: KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR
            value: "${KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR}"
          - name: KAFKA_METRICS_ENABLED
            value: "${KAFKA_METRICS_ENABLED}"
          resources: {}
          terminationMessagePath: /dev/termination-log
          terminationMessagePolicy: File
          volumeMounts:
          - mountPath: /var/lib/kafka/
            name: kafka-storage
          - mountPath: /opt/prometheus/config/
            name: kafka-metrics-config
          livenessProbe:
            exec:
              command:
              - "/opt/kafka/kafka_healthcheck.sh"
            initialDelaySeconds: ${ZOOKEEPER_HEALTHCHECK_DELAY}
            timeoutSeconds: ${ZOOKEEPER_HEALTHCHECK_TIMEOUT}
          readinessProbe:
            exec:
              command:
              - "/opt/kafka/kafka_healthcheck.sh"
            initialDelaySeconds: ${ZOOKEEPER_HEALTHCHECK_DELAY}
            timeoutSeconds: ${ZOOKEEPER_HEALTHCHECK_TIMEOUT}
        dnsPolicy: ClusterFirst
        restartPolicy: Always
        volumes:
        - name: kafka-metrics-config-${KAFKA_IMAGE_NAME}
          configMap:
            name: kafka-metrics-config-${KAFKA_IMAGE_NAME}
    volumeClaimTemplates:
    - metadata:
        name: kafka-storage
      spec:
        accessModes:
        - ReadWriteOnce
        resources:
          requests:
            storage: "${KAFKA_VOLUME_CAPACITY}"
- apiVersion: v1
  kind: ConfigMap
  metadata:
    name: zookeeper-metrics-config-${ZOOKEEPER_IMAGE_NAME}
  data:
    config.yml: |
      lowercaseOutputName: true
- apiVersion: apps/v1beta1
  kind: StatefulSet
  metadata:
    labels:
      name: zookeeper-${ZOOKEEPER_IMAGE_NAME}
    name: zookeeper-${ZOOKEEPER_IMAGE_NAME}
  spec:
    replicas: ${ZOOKEEPER_NODE_COUNT}
    selector:
      matchLabels:
        name: zookeeper-${ZOOKEEPER_IMAGE_NAME}
    serviceName: zookeeper-headless-${ZOOKEEPER_IMAGE_NAME}
    template:
      metadata:
        labels:
          name: zookeeper-${ZOOKEEPER_IMAGE_NAME}
        annotations:
          prometheus.io/scrape: "true"
          prometheus.io/path: "/metrics"
          prometheus.io/port: "9404"
      spec:
        containers:
        - name: zookeeper-${ZOOKEEPER_IMAGE_NAME}
          image: ${IMAGE_REPO_NAME}/${ZOOKEEPER_IMAGE_NAME}:${ZOOKEEPER_IMAGE_TAG}
          imagePullPolicy: Always
          ports:
          - name: clientport
            containerPort: 2181
            protocol: TCP
          - name: clustering
            containerPort: 2888
            protocol: TCP
          - name: leaderelection
            containerPort: 3888
            protocol: TCP
          - name: kafkametrics
            containerPort: 9404
            protocol: TCP
          env:
          - name: ZOOKEEPER_NODE_COUNT
            value: ${ZOOKEEPER_NODE_COUNT}
          - name: ZOOKEEPER_METRICS_ENABLED
            value: "${ZOOKEEPER_METRICS_ENABLED}"
          volumeMounts:
          - mountPath: /var/lib/zookeeper
            name: zookeeper-storage
          - mountPath: /opt/prometheus/config/
            name: zookeeper-metrics-config
          livenessProbe:
            exec:
              command:
              - "/opt/zookeeper/zookeeper_healthcheck.sh"
            initialDelaySeconds: ${KAFKA_HEALTHCHECK_DELAY}
            timeoutSeconds: ${KAFKA_HEALTHCHECK_TIMEOUT}
          readinessProbe:
            exec:
              command:
              - "/opt/zookeeper/zookeeper_healthcheck.sh"
            initialDelaySeconds: ${KAFKA_HEALTHCHECK_DELAY}
            timeoutSeconds: ${KAFKA_HEALTHCHECK_TIMEOUT}
        dnsPolicy: ClusterFirst
        restartPolicy: Always
        volumes:
        - name: zookeeper-metrics-config-${ZOOKEEPER_IMAGE_NAME}
          configMap:
            name: zookeeper-metrics-config-${ZOOKEEPER_IMAGE_NAME}
    volumeClaimTemplates:
    - metadata:
        name: zookeeper-storage
      spec:
        accessModes:
        - ReadWriteOnce
        resources:
          requests:
            storage: "${ZOOKEEPER_VOLUME_CAPACITY}"