kafka broker setup

playbooks/roles/confluent-community-broker/tasks/main.yml

@@ -1,128 +1,12 @@
 ---
-- name: Load OS specific variables
-  include_vars: "{{ item }}"
-  with_first_found:
-    - files:
-        - "{{ ansible_os_family|lower }}-{{ ansible_distribution_major_version }}.yml"
-        - "{{ ansible_os_family|lower }}-{{ ansible_distribution|lower }}.yml"
-        - "{{ ansible_os_family|lower }}.yml"
-        - defaults.yml
-      paths:
-        - ../vars
-
-- include_vars: group_vars/confluent-community
-  when: distro == "cpc"
-
-- name: install confluent community repo
-  copy:
-    src: confluent.repo
-    dest: /etc/yum.repos.d/
-  notify: yum-clean-metadata
-
-- name: add gpg key
-  rpm-key:
-    state: present
-    key: https://packages.confluent.io/rpm/5.3/archive.key
-
-- name: clean yum
-  yum: 
-
-- name: Ensure required packages are installed (yum)
-  yum:
-    name: "{{ item }}"
-    update_cache: yes
-    state: installed
-  with_items: "{{ packages|default([]) }}"
-  when: ansible_os_family == "RedHat"
-
-- name: Ensure required packages are installed (apt)
-  apt:
-    name: "{{ item }}"
-    update_cache: yes
-    state: installed
-  with_items: "{{ packages|default([]) }}"
-  when: ansible_os_family == "Debian"
-
-- name: Upgrade all packages (yum)
-  yum: name=* state=latest
-  when: ansible_os_family == "RedHat"
-
-- name: Upgrade all packages (apt)
-  apt: upgrade=dist
-  when: ansible_os_family == "Debian"
-
-- name: Set nofile limits
-  lineinfile: dest=/etc/security/limits.conf
-              insertbefore="^# End of file"
-              state=present
-              line="{{ item }}"
-  with_items:
-    - "* soft nofile 32768"
-    - "* hard nofile 32768"
-  when: not azure
-
-- name: Set nproc limits
-  lineinfile: dest=/etc/security/limits.d/90-nproc.conf
-              insertafter=EOF
-              state=present
-              create=yes
-              line="{{ item }}"
-              mode=0644
-  with_items:
-    - "* soft nproc 32768"
-    - "* hard nproc 32768"
-  when: not azure
-
-- name: Set swappiness to 1
-  sysctl: name=vm.swappiness value=1 state=present ignoreerrors=yes
-
-- name: Set the tuned profile
-  copy: src=tuned.conf
-        dest=/etc/tuned/kafka/
-        mode=0755
-  when: ansible_os_family == "RedHat" and ansible_distribution_major_version == "7"
-
-- name: Activate the tuned profile
-  shell: tuned-adm profile kafka
-  when: ansible_os_family == "RedHat" and ansible_distribution_major_version == "7"
-
-- name: Get number of kernels in grub.conf
-  shell: grep -E "^[[:blank:]]*kernel" /boot/grub/grub.conf | grep -v transparent_hugepage; exit 0
-  register: grep_result
-  when: ansible_os_family == "RedHat" and (ansible_distribution == "Amazon" or ansible_distribution_major_version == "6") and not azure
-  ignore_errors: true
-
-- name: Disable Transparent Huge Pages in Grub 1
-  lineinfile: dest=/boot/grub/grub.conf
-              backrefs=True
-              state=present
-              regexp='(^\s*kernel(\s+(?!transparent_hugepage=never)[\w=/\-\.\,]+)*)\s*$'
-              line='\1 transparent_hugepage=never'
-  with_items: "{{ grep_result.stdout_lines | default('') }}"
-  when: ansible_os_family == "RedHat" and (ansible_distribution == "Amazon" or ansible_distribution_major_version == "6") and not azure
-
-
-- name: Disable Transparent Huge Pages in Grub 2
-  lineinfile: dest=/etc/default/grub
-              state=present
-              line='GRUB_CMDLINE_LINUX=$GRUB_CMDLINE_LINUX" transparent_hugepage=never"'
-  when: ansible_distribution_major_version|int > 6 and not azure
-  notify: Run update-grub
-
-- meta: flush_handlers
-
-- name: Disable Transparent Huge Pages until reboot
-  shell: echo never > /sys/kernel/mm/transparent_hugepage/enabled && echo never > /sys/kernel/mm/transparent_hugepage/defrag
-  ignore_errors: true
-  when: not azure
-
-- name: Reconfigure resolv.conf search
-  lineinfile: dest={{ resolv_conf }}
-              create=yes
-              regexp='^search\s+(?! {{ ansible_domain }} ).*$'
-              line='search {{ ansible_domain }}'
-  when: ansible_domain != "" and not use_dns
-  notify: Run resolvconf
-
-- meta: flush_handlers
-
+- name: Set Broker Id
+  set_fact: broker_id={{item.0 + 1}}
+  with_indexed_items: "{{ groups['kafka-server-nodes'] }}"
+  when: item.1 == "{{inventory_hostname}}"
+
+- name: Template kafka server properties 
+  template:
+    src: server.properties
+    dest: /etc/kafka/
+
+- debug: var=ansible_facts

playbooks/roles/confluent-community-broker/templates/consumer.properties

@@ -16,7 +16,7 @@
 
 # list of brokers used for bootstrapping knowledge about the rest of the cluster
 # format: host1:port1,host2:port2 ...
-bootstrap.servers=104.130.220.120:9092,104.130.220.119:9092,104.130.220.122:9092
+bootstrap.servers={% for node in groups['kafka-server-nodes'] %}{{ node }}:9092{% if not loop.last %},{% endif %}{% endfor %}
 
 # consumer group id
 group.id=test-consumer-group

playbooks/roles/confluent-community-broker/templates/kafka-rest.properties

@@ -1,4 +1,5 @@
 # Maintained by Ansible
-bootstrap.servers=104.130.220.120:9092,104.130.220.119:9092,104.130.220.122:9092
+#bootstrap.servers=104.130.220.120:9092,104.130.220.119:9092,104.130.220.122:9092
+bootstrap.servers={% for node in groups['kafka-server-nodes'] %}{{ node }}:9092{% if not loop.last %},{% endif %}{% endfor %}
 listeners=http://0.0.0.0:8082
 client.ssl.endpoint.identification.algorithm=

playbooks/roles/confluent-community-broker/templates/ksql-server.properties

@@ -1,8 +1,10 @@
 # Maintained by Ansible
 
-bootstrap.servers=104.130.220.120:9092,104.130.220.119:9092,104.130.220.122:9092
+#bootstrap.servers=104.130.220.120:9092,104.130.220.119:9092,104.130.220.122:9092
+bootstrap.servers={% for node in groups['kafka-server-nodes'] %}{{ node }}:9092{% if not loop.last %},{% endif %}{% endfor %}
 
-ksql.schema.registry.url=http://104.130.220.122:8081/
+#ksql.schema.registry.url=http://104.130.220.122:8081/
+ksql.schema.registry.url=http://{{ hostvars[groups['kafka-ksql-schema-node'][0]][['ansible_', hostvars[groups['kafka-ksql-schema-node'][0]]['cluster_interface']]|join]['ipv4']['address'] }}:8081/
 
 application.id=ksql-server
 

playbooks/roles/confluent-community-broker/templates/producer.properties

@@ -18,7 +18,7 @@
 
 # list of brokers used for bootstrapping knowledge about the rest of the cluster
 # format: host1:port1,host2:port2 ...
-bootstrap.servers=104.130.220.120:9092,104.130.220.119:9092,104.130.220.122:9092
+bootstrap.servers={% for node in groups['kafka-server-nodes'] %}{{ node }}:9092{% if not loop.last %},{% endif %}{% endfor %}
 
 # specify the compression codec for all data generated: none, gzip, snappy, lz4, zstd
 compression.type=none

playbooks/roles/confluent-community-broker/templates/server.properties

@@ -0,0 +1,41 @@
+# Maintained by Ansible
+listeners=PLAINTEXT://:9092
+#listeners=104.130.220.120:9092,104.130.220.119:9092,104.130.220.122:9092
+
+#{% for node in groups['kafka-server-nodes'] %}
+#listeners={{ hostvars[node]['ansible_'~hostvars[node].cluster_interface|default(hostvars[node].ansible_default_ipv4.alias)]['ipv4']['address'] }}:2888:3888
+#{% endfor %}
+
+listeners={% for node in groups['kafka-server-nodes'] %}{{ node }}:9092{% if not loop.last %},{% endif %}{% endfor %}
+
+#zookeeper.connect=104.130.220.120:2181,104.130.220.119:2181,104.130.220.122:2181
+zookeeper.connect={% for node in groups['kafka-zk-nodes'] %}{{ node }}:2181{% if not loop.last %},{% endif %}{% endfor %}
+
+log.dirs=/kafka/data
+change to proper mount
+broker.id={{ broker_id }}
+
+log.segment.bytes=1073741824
+socket.receive.buffer.bytes=102400
+socket.send.buffer.bytes=102400
+confluent.metrics.reporter.topic.replicas=3
+num.network.threads=8
+ssl.endpoint.identification.algorithm=
+num.io.threads=16
+confluent.metrics.reporter.ssl.endpoint.identification.algorithm=
+transaction.state.log.min.isr=2
+zookeeper.connection.timeout.ms=6000
+offsets.topic.replication.factor=3
+socket.request.max.bytes=104857600
+log.retention.check.interval.ms=300000
+group.initial.rebalance.delay.ms=0
+#metric.reporters=io.confluent.metrics.reporter.ConfluentMetricsReporter
+num.recovery.threads.per.data.dir=2
+transaction.state.log.replication.factor=3
+#confluent.metrics.reporter.bootstrap.servers=104.130.220.116:9092
+log.retention.hours=168
+num.partitions=1
+
+# Confluent Support
+#confluent.support.metrics.enable=true
+#confluent.support.customer.id=anonymous