Merge pull request #15 from 116davinder/feature/test-upgrade

Feature/test upgrade

Author: 116davinder

.gitignore

@@ -3,4 +3,6 @@
 .ansible*
 *.vscode
 terraform/*/.terraform
-terraform/*/*tfstate*
+terraform/*/*tfstate*
+*.gz
+*.tgz

Readme.md

@@ -25,15 +25,18 @@ It is a group of playbooks to manage apache kafka.
 * **Zookeeper Installation**
 https://github.com/116davinder/zookeeper-cluster-ansible
 
-# **Production Environment Setup**
+## **Development Setup with Vagrant**
 
-## **Cloud Infra Setup for Apache Kafka Using Terraform**
+### **Vagrant PreSetup for cluster**
+It will update `/etc/hosts` with required dns names.
+
+```ansible-playbook -i inventory/development/cluster.ini clusterVagrantDnsUpdater.yml```
+
+## **Development with Cloud Infra Setup for Apache Kafka Using Terraform**
 
 * `terraform/aws`
 * `terraform/oci`
 
-## **Apache Kafka Playbooks**
-
 ### **AWS Cloud PreSetup for cluster**
 It will enable following things on all nodes.
 
@@ -46,6 +49,7 @@ It will enable following things on all nodes.
 
 ```ansible-playbook -i inventory/<environment>/cluster.ini clusterAwsPreSetup.yml```
 
+## **Production Environment Setup**
 
 ### **To start new cluster**
 * Update Required vars in ```inventory/<environment>/group_vars/all.yml``` .
@@ -183,11 +187,11 @@ It will be installed in similar way to apache kafka but it will start apache kaf
 ### **Tested OS**
 * CentOS 7
 * RedHat 7
-* Ubuntu / Debain ( under progress, might work :) )
+* Ubuntu
 * Amzaon Linux 2 ( under progress, might work :) )
 
 ### **Tested Ansible Version**
 ```
-ansible: 3.2.0
-ansible-base: 2.10.7
+ansible: 6.1.0
+ansible-base: 2.13.2
 ```

Vagrantfile

@@ -1,28 +1,27 @@
 Vagrant.configure("2") do |config|
 
-######################### Cluster Nodes
+# Cluster Nodes
   cluster_nodes=3
 
     (1..cluster_nodes).each do |i|
       config.vm.define "kafka-#{i}" do |node|
-        node.vm.box = "centos/7"
+        node.vm.box = "ubuntu/focal64"
         node.vm.hostname  = "kafka#{i}"
         node.vm.network :private_network, ip: "192.168.56.10#{i}"
-        #node.vm.provision :hosts, :sync_hosts => true
+        # expose JMX port
+        node.vm.network "forwarded_port", guest: 9999, host: "1000#{i}", protocol: "tcp"
+        # node.vm.provision :hosts, :sync_hosts => true
       end
   end
-##################### Setting CPU and Memory for All machines
+  # Setting CPU and Memory for All machines
   config.vm.provider "virtualbox" do |vb|
     vb.gui = false
-    vb.memory = "750"
+    vb.memory = "1024"
     vb.cpus =  1
+    vb.customize [ "modifyvm", :id, "--uartmode1", "disconnected" ] # used for wsl2
   end
 
-### Sharing Folder with machines
-### is not requried if using linux box
-  # config.vm.synced_folder ".", "/home/vagrant/projects",
-  #   mount_options: ["dmode=775,fmode=664"]
-
 # SSH config to use your local ssh key for auth instead of username/password
-    config.ssh.insert_key = false
+  config.ssh.insert_key = false
   config.vm.provision "file", source: "~/.ssh/id_rsa.pub", destination: "~/.ssh/authorized_keys"
+end

clusterJmxMonitoringSetup.yml

@@ -4,30 +4,17 @@
   gather_facts: true
   become: true
   tasks:
-    - name: install gcc and python-devel ( Redhat / CentOS )
-      ansible.builtin.yum:
-        name: "{{ item }}"
-        state: present
-      loop:
-        - gcc
-        - python3-devel
-      when: ansible_distribution in ['CentOS', 'RedHat'] | list
+    - name: install common utils
+      ansible.builtin.include_role:
+        name: common
+        tasks_from: commonUtils.yml
 
-    - name: install gcc and python-devel ( Debain / Ubuntu )
-      ansible.builtin.apt:
-        name: "{{ item }}"
-        state: present
-      loop:
-        - gcc
-        - python3-dev
-      when: ansible_distribution in ['Debian', 'Ubuntu'] | list
-
-    - name: copy kafka-python package
+    - name: copy python requirements.txt
       ansible.builtin.copy:
         src: "roles/jmxMonitor/files/requirements.txt"
         dest: "/tmp/requirements.txt"
 
-    - name: installing kafka_python with pip3
+    - name: installing requirements.txt with pip3
       ansible.builtin.pip:
         requirements: "/tmp/requirements.txt"
         executable: pip3

clusterVagrantDnsUpdater.yml

@@ -0,0 +1,13 @@
+---
+
+- hosts: clusterNodes:clusterAddNodes
+  gather_facts: true
+  become: true
+  tasks:
+    - name: "Update the /etc/hosts file with node name"
+      ansible.builtin.lineinfile:
+        dest: "/etc/hosts"
+        line: "{{ item }}\t{{ hostvars[item]['ansible_fqdn']}}"
+        state: "present"
+        backup: true
+      loop: "{{ query('inventory_hostnames', 'clusterNodes:clusterAddNodes') }}"

collections.yml

@@ -2,10 +2,10 @@
 
 collections:
   - name: 'community.general'
-    version: '3.1.0'
+    version: '4.8.0'
   - name: 'amazon.aws'
-    version: '1.5.0'
+    version: '3.3.1'
   - name: 'community.aws'
-    version: '1.5.0'
+    version: '3.4.0'
   - name: 'ansible.posix'
-    version: '1.2.0'
+    version: '1.4.0'

inventory/development/cluster.ini

@@ -2,21 +2,21 @@
 ansible_connection=ssh 
 ansible_become_method=sudo 
 ansible_become=true
+ansible_user=vagrant
 #ansible_ssh_private_key_file = ~/.vagrant.d/insecure_private_key
 
 [clusterNodes]
-10.0.0.[1:3] kafkaBrokerRackId=1
+192.168.56.10[1:3] kafkaBrokerRackId=1
 
 [clusterAddNodes]
 
 [clusterRemoveNodes]
-10.0.0.[1:3]
+192.168.56.10[1:3]
 
 [kafka-manager]
-10.0.0.1
+192.168.56.101
 
 [kafka-mirror-maker]
-10.0.0.[1:3]
 
 [kafka-mirror-maker-remove-nodes]
-10.0.0.[1:3]
+192.168.56.10[1:3]

inventory/development/group_vars/all.yml

@@ -14,32 +14,37 @@ kafkaInstallDir: /kafka
 kafkaDataDir: "{{ kafkaInstallDir }}/kafka-data"
 kafkaLogDir: "{{ kafkaInstallDir }}/kafka-logs"
 
+kafkaGcLogs: true
+kafkaGcLogsLocation: "{{ kafkaLogDir }}"
+kafkaGcLogFileCount: 5
+kafkaGcLogFileSize: 10M # with optional K, M or G suffix
+
 # JVM Settings
 # total memory * 0.25 or 25% of total memory only
 # Rest 75% will be used for Caching 
-kafkaXms: 6G                                                    # 6GB is max recommended for production kafka
+kafkaXms: "256m"             # 6GB is max recommended for production kafka
 kafkaXmx: "{{ kafkaXms }}"
 kafkaPort: 9092
 kafkaJmxPort: 9999
 kafkaVmMaxMapCount: 100000
 
 # zookeeper string should comma separate value
 # 192.168.56.100:2181,192.168.56.101:2181,192.168.56.102:2181
-kafkaZookeeperAddressString: 35.172.235.154:2181
+kafkaZookeeperAddressString: 192.168.56.101:2181
 kafkaZookeeperConnectionTimeoutMs: 10000
 
 # please use name like <environment>-kafka, so it get distinct name in NewRelic
 kafkaClusterName: development-kafka
 
-kafkaVersion: 2.8.0
+kafkaVersion: 3.2.1
 kafkaScalaVersion: 2.13
-kafkaTarLocation: "/home/dpal/Downloads/kafka_{{ kafkaScalaVersion }}-{{ kafkaVersion }}.tgz"
+kafkaTarLocation: "~/projects/kafka-cluster-ansible/kafka_{{ kafkaScalaVersion }}-{{ kafkaVersion }}.tgz"
 
 # below will be used while decommission/downgrade only
-kafkaOldVersion: 2.7.0
+kafkaOldVersion: 3.0.0
 kafkaOldScalaVersion: 2.13
 
-### Production Optimization Parameters
+### Production Optimized Parameters
 ### if nothing is set then it will use default values.
 kafkaDefaultReplicationFactor: 3
 kafkaMinInsyncReplicas: "{{ kafkaDefaultReplicationFactor | int - 1 }}"
@@ -61,13 +66,16 @@ kafkaTransactionStateLogMinIsr: "{{  kafkaOffsetsTopicReplicationFactor }}"
 kafkaLogFlushIntervalMessages: 10000
 kafkaLogFlushIntervalMs: 1000
 kafkaLogRetentionHours: 168
-kafkaLogSegmentBytes: 2073741824                                # need to ask expert kafka, should we use default 1GB here or 2GB or more
+kafkaLogSegmentBytes: 2073741824                                # need to ask kafka expert, should we use default 1GB here or 2GB or more
 kafkalogRetentionCheckIntervalMs: 300000
 kafkaGroupInitRebalanceDelayMs: 3
 
 
 # Only for AWS Based Cluster
 aws_kafka_ec2_region: "us-east-1"
 aws_kafka_ebs_device: "/dev/nvme1n1"
-aws_kafka_ebs_device_fs: "xfs"                                # currently tested with xfs only.
-aws_kafka_ebs_device_mount_location: "{{ kafkaInstallDir }}"
+aws_kafka_ebs_device_fs: "xfs"                                  # currently tested with xfs only.
+aws_kafka_ebs_device_mount_location: "{{ kafkaInstallDir }}"
+
+# Only AWS Cloudwatch Agent
+aws_kafka_cloudwatch_log_group: "kafka-{{ kafkaClusterName }}-logs"

inventory/development/group_vars/kafka-manager.yml

@@ -1,7 +1,7 @@
 ---
 
 kafkaManagerInstallDir: /opt
-kafkaManagerVersion: 3.0.0.5
+kafkaManagerVersion: 3.0.0.6
 kafkaManagerDownloadLink: "https://github.com/yahoo/CMAK/releases/download/{{ kafkaManagerVersion }}/cmak-{{ kafkaManagerVersion }}.zip"
 # if you don't want to download with ansible
 kafkaManagerRemoteDownload: true

roles/cloudwatch/tasks/main.yml

@@ -17,7 +17,7 @@
       buffer_duration = 5000
       log_stream_name = {instance_id}
       initial_position = start_of_file
-      log_group_name = kafka-{{ kafkaClusterName }}-logs
+      log_group_name = {{ aws_kafka_cloudwatch_log_group }}
 
 - name: update aws region of clouwatch agent
   ansible.builtin.lineinfile: