* updated crons

* added support for 2.7.0
* added support for 2.8.0
* updated ansible  collections

Author: 116davinder

clusterAwsPreSetup.yml

@@ -1,6 +1,6 @@
 ---
 
-- hosts: clusterNodes
+- hosts: clusterNodes:clusterAddNodes
   gather_facts: true
   become: true
   tasks:

collections.yml

@@ -2,10 +2,10 @@
 
 collections:
   - name: 'community.general'
-    version: '2.5.1'
+    version: '3.1.0'
   - name: 'amazon.aws'
-    version: '1.4.1'
+    version: '1.5.0'
   - name: 'community.aws'
-    version: '1.4.0'
+    version: '1.5.0'
   - name: 'ansible.posix'
     version: '1.2.0'

inventory/development/cluster-aws.ini

@@ -6,12 +6,18 @@ ansible_become=true
 ansible_ssh_private_key_file=~/.ssh/davinder-rnd-terraform.pem
 
 [clusterNodes]
-3.239.238.66
+18.214.25.118
 
 [clusterAddNodes]
+54.84.131.96
 
 [clusterRemoveNodes]
+54.84.131.96
+18.214.25.118
 
 [kafka-mirror-maker]
 
 [kafka-mirror-maker-remove-nodes]
+
+[kafka-manager]
+54.84.131.96

inventory/development/group_vars/all.yml

@@ -31,12 +31,12 @@ kafkaZookeeperConnectionTimeoutMs: 10000
 # please use name like <environment>-kafka, so it get distinct name in NewRelic
 kafkaClusterName: development-kafka
 
-kafkaVersion: 2.6.0
+kafkaVersion: 2.8.0
 kafkaScalaVersion: 2.13
 kafkaTarLocation: "/home/dpal/Downloads/kafka_{{ kafkaScalaVersion }}-{{ kafkaVersion }}.tgz"
 
 # below will be used while decommission/downgrade only
-kafkaOldVersion: 2.5.0
+kafkaOldVersion: 2.7.0
 kafkaOldScalaVersion: 2.13
 
 ### Production Optimization Parameters
@@ -68,6 +68,6 @@ kafkaGroupInitRebalanceDelayMs: 3
 
 # Only for AWS Based Cluster
 aws_kafka_ec2_region: "us-east-1"
-aws_kafka_ebs_device: "/dev/xvdc"
+aws_kafka_ebs_device: "/dev/nvme1n1"
 aws_kafka_ebs_device_fs: "xfs"                                # currently tested with xfs only.
 aws_kafka_ebs_device_mount_location: "{{ kafkaInstallDir }}"

inventory/development/group_vars/kafka-manager.yml

@@ -4,7 +4,7 @@ kafkaManagerInstallDir: /opt
 kafkaManagerVersion: 3.0.0.5
 kafkaManagerDownloadLink: "https://github.com/yahoo/CMAK/releases/download/{{ kafkaManagerVersion }}/cmak-{{ kafkaManagerVersion }}.zip"
 # if you don't want to download with ansible
-kafkaManagerRemoteDownload: yes
+kafkaManagerRemoteDownload: true
 kafkaManagerLocalLink: "/tmp/ansible-server/cmak-{{ kafkaManagerVersion }}.zip"
 
 kafkaManagerZkHosts: "10.0.0.1:2181,10.0.0.2:2181,10.0.0.3:2181"

roles/jmxMonitor/tasks/kafka-consumer-group-metric.yml

@@ -2,11 +2,15 @@
   ansible.builtin.file:
     path: "{{ kafkaInstallDir }}/jmxMonitor"
     state: directory
+    owner: "{{ kafkaUser }}"
+    group: "{{ kafkaGroup }}"
 
 - name: copying script and input files
   ansible.builtin.copy:
     src: "{{ item }}"
     dest: "{{ kafkaInstallDir }}/jmxMonitor/{{ item }}"
+    owner: "{{ kafkaUser }}"
+    group: "{{ kafkaGroup }}"
   loop:
     - kafka-consumer-group-metric.py
     - kafka-consumer-group-metric-input.txt
@@ -17,5 +21,5 @@
     minute: "*"
     hour: "*"
     weekday: "*"
-    user: root
+    user: "{{ kafkaUser }}"
     job: 'python3 {{ kafkaInstallDir }}/jmxMonitor/kafka-consumer-group-metric.py {{ kafkaInstallDir }}/jmxMonitor/kafka-consumer-group-metric-input.txt {{ kafkaLogDir }}/ {{ kafkaClusterName }}'

roles/jmxMonitor/tasks/kafka-mirror-maker.yml

@@ -4,11 +4,15 @@
   ansible.builtin.file:
     path: "{{ kafkaInstallDir }}/jmxMonitor"
     state: directory
+    owner: "{{ kafkaUser }}"
+    group: "{{ kafkaGroup }}"
 
 - name: copying script and input files
   ansible.builtin.copy:
     src: "{{ item }}"
     dest: "{{ kafkaInstallDir }}/jmxMonitor/{{ item }}"
+    owner: "{{ kafkaUser }}"
+    group: "{{ kafkaGroup }}"
   loop:
     - kafka-jmx-metric-collector-mm.py
     - kafka-mirror-input.txt
@@ -19,6 +23,6 @@
     minute: "*"
     hour: "*"
     weekday: "*"
-    user: root
+    user: "{{ kafkaUser }}"
     job: 'find /bin/ -name "python3*m" -print0 -exec {} {{ kafkaInstallDir }}/jmxMonitor/kafka-jmx-metric-collector-mm.py {{ ansible_fqdn }} {{ kafkaMirrorMakerJmxInitialPort + item }} {{ kafkaInstallDir }}/jmxMonitor/kafka-mirror-input.txt {{ kafkaLogDir }}/Kafka-Mirror-Maker-Process-{{ item }}- {{ kafkaClusterName }} Kafka-Mirror-Maker-Process-{{ item }} \;'
   loop: "{{ range(1, kafkaMirrorMakerProcessCountPerNode + 1, 1) | list }}"

roles/jmxMonitor/tasks/main.yml

@@ -4,11 +4,15 @@
   ansible.builtin.file:
     path: "{{ kafkaInstallDir }}/jmxMonitor"
     state: directory
+    owner: "{{ kafkaUser }}"
+    group: "{{ kafkaGroup }}"
 
 - name: copying script and input files
   ansible.builtin.copy:
     src: "{{ item }}"
     dest: "{{ kafkaInstallDir }}/jmxMonitor/{{ item }}"
+    owner: "{{ kafkaUser }}"
+    group: "{{ kafkaGroup }}"
   loop:
     - kafka-jmx-metric-collector.py
     - kafka-input.txt
@@ -19,5 +23,5 @@
     minute: "*"
     hour: "*"
     weekday: "*"
-    user: root
+    user: "{{ kafkaUser }}"
     job: 'find /bin/ -name "python3*m" -print0 -exec {} {{ kafkaInstallDir }}/jmxMonitor/kafka-jmx-metric-collector.py {{ ansible_fqdn }} {{ kafkaJmxPort }} {{ kafkaInstallDir }}/jmxMonitor/kafka-input.txt {{ kafkaLogDir }}/ {{ kafkaClusterName }} \;'

roles/kafka-manager/tasks/main.yml

@@ -1,15 +1,27 @@
 ---
 
-- name: copy and unarchive kafka manager | {{ kafkaManagerVersion }}
+- name: copy and unarchive kafka manager | {{ kafkaManagerVersion }} | Remote Download
   ansible.builtin.unarchive:
     src: "{{ kafkaManagerDownloadLink }}"
     dest: "{{ kafkaManagerInstallDir }}"
     owner: "{{ kafkaUser }}"
     group: "{{ kafkaGroup }}"
     validate_certs: false
     keep_newer: yes
-    remote_src: "{{ kafkaManagerRemoteDownload }}"
+    remote_src: true
     creates: "{{ kafkaManagerInstallDir }}/cmak-{{ kafkaManagerVersion }}/bin"
+  when: kafkaManagerRemoteDownload
+
+- name: copy and unarchive kafka | {{ kafkaManagerVersion }} | Local Upload
+  ansible.builtin.unarchive:
+    src: "{{ kafkaManagerLocalLink }}"
+    dest: "{{ kafkaManagerInstallDir }}"
+    owner: "{{ kafkaUser }}"
+    group: "{{ kafkaGroup }}"
+    validate_certs: false
+    keep_newer: yes
+    creates: "{{ kafkaManagerInstallDir }}/cmak-{{ kafkaManagerVersion }}/bin"
+  when: not kafkaManagerRemoteDownload
 
 - name: Ensuring permissions on unarchieved files
   ansible.builtin.file:

terraform/aws/var.tf

@@ -39,7 +39,7 @@ variable "allowed_inbound_client_cidrs" {
 
 variable "instance_type" {
   type        = string
-  default     = "t2.micro"
+  default     = "t3a.xlarge"
   description = "Instance Type"
 }
 
@@ -51,7 +51,7 @@ variable "key_name" {
 
 variable "kafka_nodes" {
   type        = number
-  default     = 1
+  default     = 2
   description = "how many nodes of kafka cluster is required?"
 }
 
@@ -86,4 +86,4 @@ variable "tags" {
     "owner"    = "Terraform"
     "software" = "Apache Kafka"
   }
-}
+}