seperating kafka deployment

confluent_community_rax.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+export RAX_CREDS_FILE=$(grep rax_credentials_file playbooks/group_vars/all|cut -d"'" -f2)
+export RAX_REGION=$(grep rax_region playbooks/group_vars/all|cut -d"'" -f2)
+VARS="${VARS} ANSIBLE_SCP_IF_SSH=y ANSIBLE_HOST_KEY_CHECKING=False"
+
+export $VARS
+ansible-playbook -f 20 -vvv -i inventory/rax.py playbooks/confluent-community.yml

playbooks/create_kafka_groups.yml

@@ -0,0 +1,67 @@
+---
+- name: Add nodes to required groups
+  hosts: localhost
+  connection: local
+  gather_facts: False
+  tasks:
+    - name: Add kafka nodes nodes to the kafka-cluster group
+      check_mode: no
+      add_host:
+        name: "{{ hostvars[item].inventory_hostname }}"
+        ansible_host: "{{ hostvars[item].ansible_host|default(hostvars[item].ansible_ssh_host) }}"
+        ansible_user: "{{ hostvars[item].ansible_user|default('root') }}"
+        ansible_ssh_pass: "{{ hostvars[item].ansible_ssh_pass|default('') }}"
+        ansible_become_user: root
+        ansible_become_pass: "{{ hostvars[item].ansible_ssh_pass|default('') }}"
+        groups: kafka-cluster
+      with_flattened:
+      - "{{ groups['kafka-server-nodes']|default([]) }}"
+      - "{{ groups['kafka-zk-nodes']|default([]) }}"
+      register: kafka-cluster    
+      when: "'kafka-cluster' not in groups or groups['kafka-cluster']|length < 1"
+
+    - name: Add kafka  nodes to the kafka-zookeeper cluster group
+      check_mode: no
+      add_host:
+        name: "{{ hostvars[item].inventory_hostname }}"
+        ansible_host: "{{ hostvars[item].ansible_host|default(hostvars[item].ansible_ssh_host) }}"
+        ansible_user: "{{ hostvars[item].ansible_user|default('root') }}"
+        ansible_ssh_pass: "{{ hostvars[item].ansible_ssh_pass|default('') }}"
+        ansible_become_user: root
+        ansible_become_pass: "{{ hostvars[item].ansible_ssh_pass|default('') }}"
+        groups: kafka-zookeeper-cluster
+      with_flattened:
+      - "{{ groups['kafka-zk-nodes']|default([]) }}"
+      register: kafka-zookeeper-cluster    
+      when: "'kafka-zookeeper-cluster' not in groups or groups['kafka-zookeeper-cluster']|length < 1"
+
+    - name: Add kafka nodes nodes to the kafka-broker-cluster group
+      check_mode: no
+      add_host:
+        name: "{{ hostvars[item].inventory_hostname }}"
+        ansible_host: "{{ hostvars[item].ansible_host|default(hostvars[item].ansible_ssh_host) }}"
+        ansible_user: "{{ hostvars[item].ansible_user|default('root') }}"
+        ansible_ssh_pass: "{{ hostvars[item].ansible_ssh_pass|default('') }}"
+        ansible_become_user: root
+        ansible_become_pass: "{{ hostvars[item].ansible_ssh_pass|default('') }}"
+        groups: kafka-broker-cluster
+      with_flattened:
+      - "{{ groups['kafka-server-nodes']|default([]) }}"
+      register: kafka-broker-cluster    
+      when: "'kafka-broker-cluster' not in groups or groups['kafka-broker-cluster']|length < 1"
+
+    - name: Add kafka nodes nodes to the kafka-manager group
+      check_mode: no
+      add_host:
+        name: "{{ hostvars[item].inventory_hostname }}"
+        ansible_host: "{{ hostvars[item].ansible_host|default(hostvars[item].ansible_ssh_host) }}"
+        ansible_user: "{{ hostvars[item].ansible_user|default('root') }}"
+        ansible_ssh_pass: "{{ hostvars[item].ansible_ssh_pass|default('') }}"
+        ansible_become_user: root
+        ansible_become_pass: "{{ hostvars[item].ansible_ssh_pass|default('') }}"
+        groups: kafka-manager
+      with_flattened:
+      - "{{ groups['kafka-server-nodes']|sort|last }}"
+      register: kafka-manager    
+      when: "'kafka-manager' not in groups or groups['kafka-manager']|length < 1"
+

playbooks/roles/confluent-community-broker/tasks/main.yml

@@ -1,4 +1,16 @@
 ---
+- include_vars: group_vars/kafka-server-nodes
+
+- name: Mount kafka disk under /kafka
+  mount: state=mounted
+         src="/dev/{{ kafka_disk }}1"
+         name="/kafka"
+         fstype="{{ data_disks_filesystem }}"
+         opts=defaults,noatime
+         dump=0
+         passno=0
+  when: kafka_disk is defined and kafka_disk and ansible_devices[kafka_disk] is defined
+
 - name: Set Broker Id
   set_fact: broker_id={{item.0 + 1}}
   with_indexed_items: "{{ groups['kafka-server-nodes'] }}"

playbooks/roles/confluent-community-common/files/tuned.conf

@@ -0,0 +1,12 @@
+#
+# tuned configuration
+#
+
+[main]
+include=throughput-performance
+
+[vm]
+transparent_hugepages=never
+
+[sysctl]
+vm.swappiness = 1

playbooks/roles/confluent-community-common/handlers/main.yml

@@ -3,3 +3,26 @@
   command: yum clean metadata
   args:
     warn: no
+
+- name: Restart iptables
+  service: name=iptables state=restarted enabled=yes
+
+- name: Reload firewalld
+  command: firewall-cmd --reload
+
+- name: Run update-grub
+  shell: "{{ update_grub }}"
+  when: ansible_distribution_major_version|int > 6
+
+- name: Run resolvconf
+  shell: resolvconf -u
+  when: ansible_os_family == "Debian"
+
+- name: Restart network interfaces
+  shell: ifdown {{ item }}; ifup {{ item }}; ifup {{ cluster_interface }}
+  with_flattened:
+    - cluster_interface
+    - bond_interfaces
+
+- name: Run depmod
+  command: depmod

playbooks/roles/confluent-community-common/tasks/bonding.yml

@@ -0,0 +1,36 @@
+---
+- name: Write bonding.conf file
+  lineinfile: dest=/etc/modprobe.d/bonding.conf line='alias {{ cluster_interface }} bonding' state=present create=yes mode=0664 owner=root group=root
+  notify: Run depmod
+
+- meta: flush_handlers
+
+- name: Make sure the interfaces.d files are included
+  lineinfile: line="source /etc/network/interfaces.d/ifcfg-*"
+              dest=/etc/network/interfaces
+              state=present
+              insertafter=EOF
+  when: ansible_os_family == "Debian"
+
+- name: Write ifcfg files for bonded interfaces
+  template: src={{ ansible_os_family|lower }}-ifcfg-eth.j2 dest={{ networking_path }}/ifcfg-{{ item }}
+  with_items: "{{ bond_interfaces }}"
+  notify: Restart network interfaces
+
+- name: Write ifcfg file for bond interface
+  template: src={{ ansible_os_family|lower }}-ifcfg-bond.j2 dest={{ networking_path }}/ifcfg-{{ cluster_interface }}
+  notify: Restart network interfaces
+
+- meta: flush_handlers
+
+- name: Test slave interfaces
+  include: slavetest.yml
+  with_items: "{{ bond_interfaces }}"
+  loop_control:
+    loop_var: slave_interface
+
+- name: Ping between all hosts on bond interface to verify network connectivity
+  command: /bin/ping -q -c 5 -i 0.2 -W 1 {{ hostvars[item]['bond_ip'] }}
+  when: bond_ip is defined and hostvars[item]['bond_ip'] is defined
+  with_items: "{{ groups['all'] }}"
+  changed_when: 0

playbooks/roles/confluent-community-common/tasks/firewall.yml

@@ -0,0 +1,42 @@
+---
+- name: Set iptables between cluster nodes
+  lineinfile: dest=/etc/sysconfig/iptables
+              insertbefore="^-A INPUT"
+              line="-A INPUT -s {{ hostvars[item]['ansible_'~hostvars[item].cluster_interface|default(hostvars[item].ansible_default_ipv4.alias)]['ipv4']['address'] }}/32 -j ACCEPT"
+              state=present
+  with_items: "{{ play_hosts }}"
+  when: ansible_os_family == "RedHat" and ansible_distribution_major_version == "6"
+  notify: Restart iptables
+
+- name: Set iptables to allow cluster access from external IPs
+  lineinfile: dest=/etc/sysconfig/iptables
+              insertbefore="^-A INPUT"
+              line="-A INPUT -s {{ item }}/32 -j ACCEPT"
+              state=present
+  with_items: "{{ cloud_config.allowed_external_ips }}"
+  when: ansible_os_family == "RedHat" and ansible_distribution_major_version == "6"
+  notify: Restart iptables
+
+- name: Set firewalld rules between cluster nodes
+  command: firewall-cmd --permanent --zone=public --add-rich-rule='rule family="ipv4" source address="{{ hostvars[item]['ansible_'~hostvars[item].cluster_interface|default(hostvars[item].ansible_default_ipv4.alias)]['ipv4']['address'] }}" accept'
+#  firewalld: source={{ hostvars[item][['ansible_', hostvars[item]['cluster_interface']]|join]['ipv4']['address'] }} state=enabled
+  with_items: "{{ play_hosts }}"
+  notify: Reload firewalld
+  when: ansible_os_family == "RedHat" and ansible_distribution_major_version == "7"
+
+- name: Set firewalld to allow cluster access from external IPs
+  command: firewall-cmd --permanent --zone=public --add-rich-rule='rule family="ipv4" source address="{{ item }}" accept'
+#  firewalld: source={{ item }} state=enabled
+  with_items: "{{ cloud_config.allowed_external_ips }}"
+  notify: Reload firewalld
+  when: ansible_os_family == "RedHat" and ansible_distribution_major_version == "7"
+
+- name: Set UFW rules between cluster nodes
+  ufw: rule=allow src={{ hostvars[item]['ansible_'~hostvars[item].cluster_interface|default(hostvars[item].ansible_default_ipv4.alias)]['ipv4']['address'] }}
+  with_items: "{{ play_hosts }}"
+  when: ansible_distribution == "Ubuntu"
+
+- name: Set UFW to allow cluster access from external IPs
+  ufw: rule=allow src={{ item }}
+  with_items: "{{ cloud_config.allowed_external_ips }}"
+  when: ansible_distribution == "Ubuntu"

playbooks/roles/confluent-community-common/tasks/main.yml

@@ -13,20 +13,21 @@
 - include_vars: group_vars/confluent-community
   when: distro == "cpc"
 
+- include_vars: group_vars/kafka-server-nodes
+
+- include_vars: group_vars/kafka-zookeeper-nodes
+
 - name: install confluent community repo
   copy:
     src: confluent.repo
     dest: /etc/yum.repos.d/
   notify: yum-clean-metadata
 
 - name: add gpg key
-  rpm-key:
+  rpm_key:
     state: present
     key: https://packages.confluent.io/rpm/5.3/archive.key
 
-- name: clean yum
-  yum: 
-
 - name: Ensure required packages are installed (yum)
   yum:
     name: "{{ item }}"
@@ -126,3 +127,21 @@
 
 - meta: flush_handlers
 
+- name: Configure bonding
+  include: bonding.yml
+  when: bond_interfaces is defined
+
+- name: Set hosts file
+  template: src=hosts.j2 dest=/etc/hosts mode=0644
+  when: not azure
+
+- name: Include firewall.yml
+  include: firewall.yml
+  when: configure_firewall or rax_id is defined
+
+- name: Include partitioning.yml
+  include: partitioning.yml
+  with_flattened:
+    - "{{ kafka_disk|default([]) }}"
+    - "{{ zookeeper_disk|default([]) }}"
+

playbooks/roles/confluent-community-common/tasks/partitioning.yml

@@ -0,0 +1,17 @@
+---
+- name: Get disk alignment for disk {{ item }}
+  shell: if [[ -e /sys/block/{{ item }}/queue/optimal_io_size && -e /sys/block/{{ item }}/alignment_offset && -e /sys/block/{{ item }}/queue/physical_block_size ]]; then echo $[$(( ($(cat /sys/block/{{ item }}/queue/optimal_io_size) + $(cat /sys/block/{{ item }}/alignment_offset)) / $(cat /sys/block/{{ item }}/queue/physical_block_size) )) | 2048]; else echo 2048; fi
+  when: ansible_devices[item] is defined
+  register: disk_offset
+
+- name: Partition disk {{ item }}
+  shell: parted -s -a optimal /dev/{{ item }} mklabel gpt mkpart primary {{ data_disks_filesystem }} {{ disk_offset.stdout|default("2048") }}s 100% && sleep 5 && partprobe /dev/{{ item }}; sleep 5
+  when: ansible_devices[item] is defined and ansible_devices[item]['partitions'] == {}
+
+- name: Create the filesystem on disk {{ item }}
+  filesystem: dev="/dev/{{ item }}1" fstype="{{ data_disks_filesystem }}"
+  when: ansible_devices[item] is defined
+
+- name: Disable periodic fsck on {{ item }}
+  shell: tune2fs -c0 -i0 /dev/{{ item }}1
+  when: ansible_devices[item] is defined and (data_disks_filesystem == "ext4" or data_disks_filesystem == "ext3")

playbooks/roles/confluent-community-common/templates/debian-ifcfg-bond.j2

@@ -0,0 +1,7 @@
+auto {{ cluster_interface }}
+iface {{ cluster_interface }} inet static
+    address {{ bond_ip }}
+    netmask {{ bond_netmask }}
+    bond-mode active-backup
+    bond-miimon 100
+    bond-slaves none

playbooks/roles/confluent-community-common/templates/debian-ifcfg-eth.j2

@@ -0,0 +1,3 @@
+auto {{ item }}
+iface {{ item }} inet manual
+    bond-master {{ cluster_interface }}

playbooks/roles/confluent-community-common/templates/hosts.j2

@@ -1,5 +1,5 @@
 127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4
 ::1 localhost localhost.localdomain localhost6 localhost6.localdomain6
-{% for node in groups['hadoop-cluster'] %}
+{% for node in groups['kafka-cluster'] %}
 {{ hostvars[node]['ansible_'~hostvars[node].cluster_interface|default(hostvars[node].ansible_default_ipv4.alias)]['ipv4']['address'] }} {{ hostvars[node]['ansible_nodename'] }} {{ hostvars[node]['ansible_hostname'] }}
 {% endfor %}

playbooks/roles/confluent-community-common/templates/mdadm.conf.j2

@@ -0,0 +1 @@
+{{ mdadm_scan.stdout }}

playbooks/roles/confluent-community-common/templates/redhat-ifcfg-bond.j2

@@ -0,0 +1,10 @@
+DEVICE={{ cluster_interface }}
+IPADDR={{ bond_ip }}
+NETMASK={{ bond_netmask }}
+NETWORK=
+BROADCAST=
+ONBOOT=yes
+BOOTPROTO=none
+USERCTL=no
+BONDING_OPTS="mode=active-backup miimon=100"
+NM_CONTROLLED=no

playbooks/roles/confluent-community-common/templates/redhat-ifcfg-eth.j2

@@ -0,0 +1,7 @@
+DEVICE={{ item }} 
+ONBOOT=yes
+BOOTPROTO=none
+USERCTL=no
+MASTER={{ cluster_interface }}
+SLAVE=yes
+NM_CONTROLLED=no

playbooks/roles/confluent-community-common/templates/redhat-unbound-master.j2

@@ -0,0 +1,19 @@
+server:
+    interface: 127.0.0.1
+    interface: {{ hostvars[groups['master-nodes'][0]][['ansible_', hostvars[groups['master-nodes'][0]]['cluster_interface']]|join]['ipv4']['address'] }}
+    access-control: 127.0.0.0/8 allow
+    access-control: 10.0.0.0/8 allow
+    domain-insecure: "{{ ansible_domain }}"
+    local-zone: "10.in-addr.arpa." nodefault
+
+stub-zone:
+    name: "10.in-addr.arpa."
+    stub-addr: {{ hostvars[groups['master-nodes'][0]][['ansible_', hostvars[groups['master-nodes'][0]]['cluster_interface']]|join]['ipv4']['address'] }}
+
+forward-zone:
+    name: "."
+    forward-addr: 8.8.8.8
+    forward-addr: 8.8.4.4
+
+remote-control:
+    control-interface: 127.0.0.1

playbooks/roles/confluent-community-common/templates/redhat-unbound-zonefile.j2

@@ -0,0 +1,9 @@
+# Local zone
+
+local-zone: "{{ ansible_domain }}." static
+
+{% for node in groups['hadoop-cluster'] %}
+local-data: "{{ hostvars[node]['ansible_fqdn'] }}. IN A {{ hostvars[node]['ansible_'~hostvars[node].cluster_interface|default(hostvars[node].ansible_default_ipv4.alias)]['ipv4']['address'] }}"
+local-data-ptr: "{{ hostvars[node]['ansible_'~hostvars[node].cluster_interface|default(hostvars[node].ansible_default_ipv4.alias)]['ipv4']['address'] }} {{ hostvars[node]['ansible_fqdn'] }}"
+
+{% endfor %}

playbooks/roles/confluent-community-common/templates/redhat-unbound.j2

@@ -0,0 +1,17 @@
+server:
+    interface: 127.0.0.1
+    access-control: 127.0.0.0/8 allow
+    access-control: 10.0.0.0/8 allow
+    domain-insecure: "{{ ansible_domain }}"
+    local-zone: "10.in-addr.arpa." nodefault
+
+stub-zone:
+    name: "10.in-addr.arpa."
+    stub-addr: {{ hostvars[groups['master-nodes'][0]][['ansible_', hostvars[groups['master-nodes'][0]]['cluster_interface']]|join]['ipv4']['address'] }}
+
+forward-zone:
+    name: "."
+    forward-addr: {{ hostvars[groups['master-nodes'][0]][['ansible_', hostvars[groups['master-nodes'][0]]['cluster_interface']]|join]['ipv4']['address'] }}
+
+remote-control:
+    control-interface: 127.0.0.1

playbooks/roles/confluent-community-common/templates/resolv.conf.j2

@@ -0,0 +1,4 @@
+domain {{ ansible_domain }}
+nameserver 173.203.4.8
+nameserver 173.203.4.9
+search {{ ansible_domain }}