Remove kafka_var hitachienergy#2803

ansible/playbooks/roles/kafka/tasks/generate-certificates.yml

@@ -1,55 +1,55 @@
 - name: Create stores directory
   file:
-    path: "{{ specification.kafka_var.security.ssl.server.keystore_location | dirname }}"
+    path: "{{ specification.security.ssl.server.keystore_location | dirname }}"
     state: directory
-    owner: "{{ specification.kafka_var.user }}"
-    group: "{{ specification.kafka_var.group }}"
+    owner: "{{ specification.user }}"
+    group: "{{ specification.group }}"
     mode: "0755"
 
 - name: Check if keystore exists on broker
   stat:
-    path: "{{ specification.kafka_var.security.ssl.server.keystore_location }}"
+    path: "{{ specification.security.ssl.server.keystore_location }}"
   changed_when: false
   register: keystore_exists
 
 - name: Generate keystore for each server
-  shell: keytool -keystore {{ specification.kafka_var.security.ssl.server.keystore_location }} \
-         -alias localhost -validity {{ specification.kafka_var.security.ssl.server.cert_validity }} -genkey -keyalg RSA \
-         -noprompt -storepass {{ specification.kafka_var.security.ssl.server.passwords.keystore }} \
-         -keypass {{ specification.kafka_var.security.ssl.server.passwords.key }} \
+  shell: keytool -keystore {{ specification.security.ssl.server.keystore_location }} \
+         -alias localhost -validity {{ specification.security.ssl.server.cert_validity }} -genkey -keyalg RSA \
+         -noprompt -storepass {{ specification.security.ssl.server.passwords.keystore }} \
+         -keypass {{ specification.security.ssl.server.passwords.key }} \
          -dname "CN={{ inventory_hostname }}" -ext SAN="DNS:{{ inventory_hostname }}"
   when:
     - not keystore_exists.stat.exists
 
 - name: Check if signing certificate exists
   stat:
-    path: "{{ specification.kafka_var.security.ssl.server.keystore_location | dirname }}/ca-cert"
+    path: "{{ specification.security.ssl.server.keystore_location | dirname }}/ca-cert"
   register: signing_certificate_exists
   changed_when: false
   when:
     - groups['kafka'][0] == inventory_hostname
 
 - name: Generate signing certificate
-  shell: openssl req -new -x509 -keyout {{ specification.kafka_var.security.ssl.server.keystore_location | dirname }}/ca-key \
-         -out {{ specification.kafka_var.security.ssl.server.keystore_location | dirname }}/ca-cert \
-         -days {{ specification.kafka_var.security.ssl.server.cert_validity }} \
+  shell: openssl req -new -x509 -keyout {{ specification.security.ssl.server.keystore_location | dirname }}/ca-key \
+         -out {{ specification.security.ssl.server.keystore_location | dirname }}/ca-cert \
+         -days {{ specification.security.ssl.server.cert_validity }} \
          -subj "/CN={{ inventory_hostname }}" \
-         --passout pass:{{ specification.kafka_var.security.ssl.server.passwords.key }}
+         --passout pass:{{ specification.security.ssl.server.passwords.key }}
   when:
     - groups['kafka'][0] == inventory_hostname
     - not signing_certificate_exists.stat.exists
 
 - name: Create kafka certificates directory on Epiphany host
   become: false
   file:
-    path: "{{ specification.kafka_var.security.ssl.server.local_cert_download_path }}"
+    path: "{{ specification.security.ssl.server.local_cert_download_path }}"
     state: directory
   delegate_to: localhost
 
 - name: Fetching files
   fetch:
-    src: "{{ specification.kafka_var.security.ssl.server.keystore_location | dirname }}/{{ item }}"
-    dest: "{{ specification.kafka_var.security.ssl.server.local_cert_download_path }}/{{ item }}"
+    src: "{{ specification.security.ssl.server.keystore_location | dirname }}/{{ item }}"
+    dest: "{{ specification.security.ssl.server.local_cert_download_path }}/{{ item }}"
     flat: yes
   loop:
     - "ca-cert"
@@ -59,8 +59,8 @@
 
 - name: Copy signing certificate and key to brokers
   copy:
-     src: "{{ specification.kafka_var.security.ssl.server.local_cert_download_path }}/{{ item }}"
-     dest: "{{ specification.kafka_var.security.ssl.server.keystore_location | dirname }}/"
+     src: "{{ specification.security.ssl.server.local_cert_download_path }}/{{ item }}"
+     dest: "{{ specification.security.ssl.server.keystore_location | dirname }}/"
   loop:
     - "ca-cert"
     - "ca-key"
@@ -69,71 +69,71 @@
 
 - name: Check if trustore exists
   stat:
-    path: "{{ specification.kafka_var.security.ssl.server.truststore_location }}"
+    path: "{{ specification.security.ssl.server.truststore_location }}"
   register: trustore_exists
 
 - name: Create trustore
-  shell: keytool -noprompt -keystore "{{ specification.kafka_var.security.ssl.server.truststore_location }}" -alias CARoot \
-         -import -file "{{ specification.kafka_var.security.ssl.server.keystore_location | dirname }}/ca-cert" \
-         -storepass {{ specification.kafka_var.security.ssl.server.passwords.keystore }} \
-         -keypass {{ specification.kafka_var.security.ssl.server.passwords.key }}
+  shell: keytool -noprompt -keystore "{{ specification.security.ssl.server.truststore_location }}" -alias CARoot \
+         -import -file "{{ specification.security.ssl.server.keystore_location | dirname }}/ca-cert" \
+         -storepass {{ specification.security.ssl.server.passwords.keystore }} \
+         -keypass {{ specification.security.ssl.server.passwords.key }}
   when:
     - not trustore_exists.stat.exists
 
 - name: Check if CA certificate is already imported
-  shell: keytool -list -v -keystore {{ specification.kafka_var.security.ssl.server.keystore_location }} \
-         -storepass {{ specification.kafka_var.security.ssl.server.passwords.keystore }} \
+  shell: keytool -list -v -keystore {{ specification.security.ssl.server.keystore_location }} \
+         -storepass {{ specification.security.ssl.server.passwords.keystore }} \
          | grep -i "Alias name" | grep -i "caroot"
   failed_when: "caroot_exists.rc == 2"
   changed_when: false
   register: caroot_exists
 
 - name: Check if certificate signed by CA is already imported
   shell: |-
-         keytool -list -v -keystore {{ specification.kafka_var.security.ssl.server.keystore_location }} \
-         -storepass {{ specification.kafka_var.security.ssl.server.passwords.keystore }} \
+         keytool -list -v -keystore {{ specification.security.ssl.server.keystore_location }} \
+         -storepass {{ specification.security.ssl.server.passwords.keystore }} \
          -alias localhost \
          | grep -i 'Certificate chain length: 2'
   failed_when: "signed_cert_exists.rc == 2"
   changed_when: false
   register: signed_cert_exists
 
 - name: Export certificate to sign certificate with CA
-  shell: keytool -noprompt -keystore {{ specification.kafka_var.security.ssl.server.keystore_location }} \
+  shell: keytool -noprompt -keystore {{ specification.security.ssl.server.keystore_location }} \
          -alias localhost -certreq \
-         -file "{{ specification.kafka_var.security.ssl.server.keystore_location | dirname }}/cert-file" \
-         -storepass {{ specification.kafka_var.security.ssl.server.passwords.keystore }} \
-         -keypass {{ specification.kafka_var.security.ssl.server.passwords.key }}
+         -file "{{ specification.security.ssl.server.keystore_location | dirname }}/cert-file" \
+         -storepass {{ specification.security.ssl.server.passwords.keystore }} \
+         -keypass {{ specification.security.ssl.server.passwords.key }}
   when:
     - signed_cert_exists.rc == 1
 
 - name: Signing certificate with CA
-  shell: openssl x509 -req -CA "{{ specification.kafka_var.security.ssl.server.keystore_location | dirname }}/ca-cert" \
-         -CAkey "{{ specification.kafka_var.security.ssl.server.keystore_location | dirname }}/ca-key" \
-         -in "{{ specification.kafka_var.security.ssl.server.keystore_location | dirname }}/cert-file" \
-         -out "{{ specification.kafka_var.security.ssl.server.keystore_location | dirname }}/cert-signed" \
-         -days {{ specification.kafka_var.security.ssl.server.cert_validity }} -CAcreateserial \
-         -passin pass:{{ specification.kafka_var.security.ssl.server.passwords.key }}
+  shell: openssl x509 -req -CA "{{ specification.security.ssl.server.keystore_location | dirname }}/ca-cert" \
+         -CAkey "{{ specification.security.ssl.server.keystore_location | dirname }}/ca-key" \
+         -in "{{ specification.security.ssl.server.keystore_location | dirname }}/cert-file" \
+         -out "{{ specification.security.ssl.server.keystore_location | dirname }}/cert-signed" \
+         -days {{ specification.security.ssl.server.cert_validity }} -CAcreateserial \
+         -passin pass:{{ specification.security.ssl.server.passwords.key }}
   when:
     - signed_cert_exists.rc == 1
 
 - name: Import certificate CA
-  shell: keytool -noprompt -keystore {{ specification.kafka_var.security.ssl.server.keystore_location }} -alias CARoot \
-         -import -file "{{ specification.kafka_var.security.ssl.server.keystore_location | dirname }}/ca-cert" \
-         -storepass {{ specification.kafka_var.security.ssl.server.passwords.keystore }}
+  shell: keytool -noprompt -keystore {{ specification.security.ssl.server.keystore_location }} -alias CARoot \
+         -import -file "{{ specification.security.ssl.server.keystore_location | dirname }}/ca-cert" \
+         -storepass {{ specification.security.ssl.server.passwords.keystore }}
   when:
     - caroot_exists.rc == 1
 
 - name: Import certificate signed by CA
-  shell: keytool -noprompt -keystore {{ specification.kafka_var.security.ssl.server.keystore_location }} -alias localhost \
-         -import -file "{{ specification.kafka_var.security.ssl.server.keystore_location | dirname }}/cert-signed" \
-         -storepass {{ specification.kafka_var.security.ssl.server.passwords.keystore }}
+  shell: keytool -noprompt -keystore {{ specification.security.ssl.server.keystore_location }} -alias localhost \
+         -import -file "{{ specification.security.ssl.server.keystore_location | dirname }}/cert-signed" \
+         -storepass {{ specification.security.ssl.server.passwords.keystore }}
   when:
     - signed_cert_exists.rc == 1
 
 - name: Remove extracted key and cert from others than root node
   file:
-    path: "{{ specification.kafka_var.security.ssl.server.keystore_location | dirname }}/{{ item }}"
+    path: "{{ specification.security.ssl.server.keystore_location | dirname }}/{{ item }}"
     state: absent
   loop:
     - "ca-cert"

ansible/playbooks/roles/kafka/tasks/metrics.yml

@@ -2,7 +2,7 @@
 
 - name: prometheus jmx | add kafka user to correct jmx exporter user
   user:
-    name: "{{ specification.kafka_var.user }}"
+    name: "{{ specification.user }}"
     groups: "{{ specification.jmx_exporter_group }}"
     append: yes
 
@@ -11,8 +11,8 @@
   copy:
     dest: "{{ specification.prometheus_jmx_config }}"
     src: jmx-kafka-config.yml
-    owner: "{{ specification.kafka_var.user }}"
-    group: "{{ specification.kafka_var.group }}"
+    owner: "{{ specification.user }}"
+    group: "{{ specification.group }}"
     mode: 0644
 
 - name: delegated | create prometheus system group

ansible/playbooks/roles/kafka/tasks/setup-kafka.yml

@@ -1,14 +1,14 @@
 ---
 - name: Setup group
   group:
-    name: "{{ specification.kafka_var.group }}"
+    name: "{{ specification.group }}"
     system: yes
 
 - name: Setup user
   user:
-    name: "{{ specification.kafka_var.user }}"
+    name: "{{ specification.user }}"
     system: yes
-    group: "{{ specification.kafka_var.group }}"
+    group: "{{ specification.group }}"
     shell: "/usr/sbin/nologin"
 
 - name: Install Java package
@@ -55,55 +55,55 @@
 
 - name: Create data_dir
   file:
-    path: "{{ specification.kafka_var.data_dir }}"
+    path: "{{ specification.data_dir }}"
     state: directory
-    owner: "{{ specification.kafka_var.user }}"
-    group: "{{ specification.kafka_var.group }}"
+    owner: "{{ specification.user }}"
+    group: "{{ specification.group }}"
     mode: 0755
 
 - name: Remove lost+found in the datadir
   file:
-    path: "{{ specification.kafka_var.data_dir }}/lost+found"
+    path: "{{ specification.data_dir }}/lost+found"
     state: absent
 
 - name: Create log_dir
   file:
-    path: "{{ specification.kafka_var.log_dir }}"
+    path: "{{ specification.log_dir }}"
     state: directory
-    owner: "{{ specification.kafka_var.user }}"
-    group: "{{ specification.kafka_var.group }}"
+    owner: "{{ specification.user }}"
+    group: "{{ specification.group }}"
     mode: 0755
 
 - name: Create /etc/kafka directory
   file:
     path: /etc/kafka
     state: directory
-    owner: "{{ specification.kafka_var.user }}"
-    group: "{{ specification.kafka_var.group }}"
+    owner: "{{ specification.user }}"
+    group: "{{ specification.group }}"
 
 # - name: link conf_dir to /opt/kafka/config
 #   file: dest=/etc/kafka owner=kafka group=kafka state=link src=/opt/kafka/config
 
 # Setup log4j.properties
 - name: Create log4j.properties
   file:
-    path: "{{ specification.kafka_var.conf_dir }}/log4j.properties"
-    owner: "{{ specification.kafka_var.user }}"
-    group: "{{ specification.kafka_var.group }}"
+    path: "{{ specification.conf_dir }}/log4j.properties"
+    owner: "{{ specification.user }}"
+    group: "{{ specification.group }}"
     mode: 0644
 
 - name: Generate certificate
   include_tasks: generate-certificates.yml
   when:
-    - specification.kafka_var.security.ssl.enabled is defined
-    - specification.kafka_var.security.ssl.enabled
+    - specification.security.ssl.enabled is defined
+    - specification.security.ssl.enabled
 
 # Setup server.properties
 - name: Create server.properties
   template:
-    dest: "{{ specification.kafka_var.conf_dir }}/server.properties"
-    owner: "{{ specification.kafka_var.user }}"
-    group: "{{ specification.kafka_var.group }}"
+    dest: "{{ specification.conf_dir }}/server.properties"
+    owner: "{{ specification.user }}"
+    group: "{{ specification.group }}"
     # Was 0640
     mode: 0644
     src: server.properties.j2
@@ -114,7 +114,7 @@
 - name: Delete meta.properties
   become: true
   file:
-    path: "{{ specification.kafka_var.data_dir }}/meta.properties"
+    path: "{{ specification.data_dir }}/meta.properties"
     state: absent
   when: create_server_properties.changed
 
@@ -128,7 +128,7 @@
 
 - name: configure system settings, file descriptors and number of threads for kafka
   pam_limits:
-    domain: "{{ specification.kafka_var.user }}"
+    domain: "{{ specification.user }}"
     limit_type: "{{ item.limit_type }}"
     limit_item: "{{ item.limit_item }}"
     value: "{{item.value}}"

ansible/playbooks/roles/kafka/templates/client-ssl.properties.j2

@@ -1,4 +1,4 @@
-bootstrap.servers={{ kafka_hosts }}:{{ specification.kafka_var.security.ssl.port }}
+bootstrap.servers={{ kafka_hosts }}:{{ specification.security.ssl.port }}
 security.protocol=SSL
 ssl.truststore.location=/var/private/ssl/kafka.client.truststore.jks
-ssl.truststore.password={{ specification.kafka_var.security.ssl.client.passwords.truststore }}
+ssl.truststore.password={{ specification.security.ssl.client.passwords.truststore }}

ansible/playbooks/roles/kafka/templates/kafka.service.j2

@@ -2,8 +2,8 @@
 Description=Kafka Daemon
 After=zookeeper.service
 
-{% if specification.kafka_var.javax_net_debug is defined %}
-{%      set javax_debug = '-Djavax.net.debug=' ~ specification.kafka_var.javax_net_debug %}
+{% if specification.javax_net_debug is defined %}
+{%      set javax_debug = '-Djavax.net.debug=' ~ specification.javax_net_debug %}
 {% else %}
 {%      set javax_debug = '' %}
 {% endif %}
@@ -14,14 +14,14 @@ User=kafka
 Group=kafka
 LimitNOFILE=32768
 Restart=on-failure
-Environment="KAFKA_HEAP_OPTS={{ specification.kafka_var.heap_opts }}"
-Environment="LOG_DIR={{ specification.kafka_var.log_dir }}"
+Environment="KAFKA_HEAP_OPTS={{ specification.heap_opts }}"
+Environment="LOG_DIR={{ specification.log_dir }}"
 {% if exporter.stat.exists %}
 Environment="KAFKA_OPTS={{ javax_debug }} -javaagent:{{ prometheus_jmx_exporter_path }}={{ specification.prometheus_jmx_exporter_web_listen_port }}:{{ specification.prometheus_jmx_config }}"
 {% else %}
 Environment="KAFKA_OPTS={{ javax_debug }}"
 {% endif %}
-Environment="KAFKA_JMX_OPTS={{ specification.kafka_var.jmx_opts }}"
+Environment="KAFKA_JMX_OPTS={{ specification.jmx_opts }}"
 ExecStart=/opt/kafka/bin/kafka-server-start.sh /opt/kafka/config/server.properties
 
 [Install]

ansible/playbooks/roles/kafka/templates/kafka_producer_consumer.py.j2

@@ -20,8 +20,8 @@ class Producer(threading.Thread):
         producer = KafkaProducer(bootstrap_servers='{{ kafka_hosts }}')
 
         while not self.stop_event.is_set():
-            {% for msg in specification.kafka_var.tests.epiphany_topic_test_msgs %}
-            producer.send('{{ specification.kafka_var.tests.epiphany_topic_test }}', b"{{ msg }}")
+            {% for msg in specification.tests.epiphany_topic_test_msgs %}
+            producer.send('{{ specification.tests.epiphany_topic_test }}', b"{{ msg }}")
             {% endfor %}
             time.sleep(1)
 
@@ -40,7 +40,7 @@ class Consumer(multiprocessing.Process):
         consumer = KafkaConsumer(bootstrap_servers='{{ kafka_hosts }}',
                                  auto_offset_reset='earliest',
                                  consumer_timeout_ms=1000)
-        consumer.subscribe(['{{ specification.kafka_var.tests.epiphany_topic_test }}'])
+        consumer.subscribe(['{{ specification.tests.epiphany_topic_test }}'])
 
         while not self.stop_event.is_set():
             for message in consumer:

ansible/playbooks/roles/kafka/templates/kafka_server_jaas.conf.j2

@@ -1,8 +1,8 @@
 KafkaServer {
    org.apache.kafka.common.security.plain.PlainLoginModule required
-   username="{{ specification.kafka_var.admin }}"
-   password="{{ specification.kafka_var.admin_pwd }}"
-   user_admin="{{ specification.kafka_var.admin_pwd }}"
+   username="{{ specification.admin }}"
+   password="{{ specification.admin_pwd }}"
+   user_admin="{{ specification.admin_pwd }}"
    {%- for host in kafka_hosts %}
    user_{{host}}="kafkabroker1-secret";
    {%- endfor %}

ansible/playbooks/roles/kafka/templates/logrotate.conf.j2

@@ -1,4 +1,4 @@
-{{ specification.kafka_var.log_dir }}/*.log {
+{{ specification.log_dir }}/*.log {
   rotate 5
   daily
   compress