Refactored repository role, changes for RHEL (#547)

* Refactored repository role, changes for RHEL #536

* Changed directories structure for tasks #536

* Added repository role to feature mappings #536

* Fix for removing flag file if expired #536

* Added wget #536

core/src/epicli/cli/engine/PrepareEngine.py

@@ -11,7 +11,7 @@
 
 
 class PrepareEngine(Step):
-    PREPARE_PATH = DATA_FOLDER_PATH + '/common/ansible/playbooks/roles/repository/files/prepare'
+    PREPARE_PATH = DATA_FOLDER_PATH + '/common/ansible/playbooks/roles/repository/files/download-requirements'
 
     def __init__(self, input_data):
         super().__init__(__name__)

core/src/epicli/cli/engine/ansible/AnsibleRunner.py

@@ -61,7 +61,7 @@ def run(self):
         self.logger.info('Setting up repository for cluster provisioning. This will take a while...')
         repository_setup_play_result = self.ansible_command.run_playbook_with_retries(
                                                                             inventory=inventory_path,
-                                                                            playbook_path=self.playbook_path('repository-setup'),
+                                                                            playbook_path=self.playbook_path('repository_setup'),
                                                                             retries=5)
 
         if repository_setup_play_result != 0:
@@ -84,7 +84,7 @@ def run(self):
 
         repository_teardown_play_result = self.ansible_command.run_playbook_with_retries(
                                                                             inventory=inventory_path,
-                                                                            playbook_path=self.playbook_path('repository-teardown'),
+                                                                            playbook_path=self.playbook_path('repository_teardown'),
                                                                             retries=1)
 
         if repository_teardown_play_result != 0:

core/src/epicli/data/common/ansible/playbooks/roles/repository/files/repository/RedHat/setup-epirepo-client.sh → core/src/epicli/data/common/ansible/playbooks/roles/repository/files/client/RedHat/add-epirepo-client.sh

@@ -1,12 +1,9 @@
-#!/bin/bash
+#!/bin/bash -eu
 
 REPOSITORY_URL=$1
 
 curl -I -L $REPOSITORY_URL | grep "HTTP/1.1 200 OK"
 
-# Todo: remove if unnecessary
-#sed -i -e "s/enabled=1/enabled=0/g" /etc/yum.repos.d/*.repo;
-
 cat << EOF > /etc/yum.repos.d/epirepo.repo
 [epirepo]
 name=epirepo

core/src/epicli/data/common/ansible/playbooks/roles/repository/files/client/RedHat/create-enabled-system-repos-list.sh

@@ -0,0 +1,7 @@
+#!/bin/bash -eu
+
+ENABLED_REPOS_FILE=/tmp/enabled-system-repos.txt
+
+if [ ! -f "$ENABLED_REPOS_FILE" ]; then
+  yum repolist -v enabled | grep -i Repo-id | awk -F ":" '{print $2}' |  sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' | awk -F "/" '{print $1}' > $ENABLED_REPOS_FILE
+fi

core/src/epicli/data/common/ansible/playbooks/roles/repository/files/client/RedHat/disable-epirepo-client.sh

@@ -0,0 +1,5 @@
+#!/bin/bash -eu
+
+yum-config-manager --disable epirepo
+yum makecache
+yum repolist

core/src/epicli/data/common/ansible/playbooks/roles/repository/files/client/RedHat/disable-system-repos.sh

@@ -0,0 +1,10 @@
+#!/bin/bash -eu
+
+REPOS_LIST_FILE=/tmp/enabled-system-repos.txt
+
+cat $REPOS_LIST_FILE | while read repository
+do
+  echo "Disabling repository: $repository"
+  yum-config-manager --disable $repository
+done
+

core/src/epicli/data/common/ansible/playbooks/roles/repository/files/client/RedHat/enable-system-repos.sh

@@ -0,0 +1,9 @@
+#!/bin/bash -eu
+
+REPOS_LIST_FILE=/tmp/enabled-system-repos.txt
+
+cat $REPOS_LIST_FILE | while read repository
+do
+  echo "Enabling repository: $repository"
+  yum-config-manager --enable $repository
+done

core/src/epicli/data/common/ansible/playbooks/roles/repository/files/prepare/centos-7/requirements.txt → core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.txt

@@ -87,6 +87,7 @@ tmux
 urw-base35-fonts # for grafana
 vim-common # for vim
 vim-enhanced
+wget
 xorg-x11-font-utils # for grafana
 xorg-x11-server-utils # for grafana
 yum-plugin-versionlock

core/src/epicli/data/common/ansible/playbooks/roles/repository/files/prepare/redhat-7/requirements.txt → core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/requirements.txt

@@ -87,6 +87,7 @@ tmux
 urw-base35-fonts # for grafana
 vim-common # for vim
 vim-enhanced
+wget
 xorg-x11-font-utils # for grafana
 xorg-x11-server-utils # for grafana
 yum-plugin-versionlock

core/src/epicli/data/common/ansible/playbooks/roles/repository/files/server/RedHat/create-repository.sh

@@ -0,0 +1,15 @@
+#!/bin/bash -eu
+
+EPI_REPO_SERVER_PATH=$1 # /var/www/html/epirepo is the default
+IS_OFFLINE_MODE=$2
+
+if $IS_OFFLINE_MODE = true; then
+  yum install -y $(ls $EPI_REPO_SERVER_PATH/packages/offline_prereqs/*.rpm) #TODO: to test
+else
+  yum install -y httpd createrepo yum-utils
+fi
+
+setenforce 0
+systemctl start httpd
+
+createrepo $EPI_REPO_SERVER_PATH/packages

core/src/epicli/data/common/ansible/playbooks/roles/repository/files/server/RedHat/disable-repository-server.sh

@@ -0,0 +1,4 @@
+#!/bin/bash -eu
+
+systemctl stop httpd
+systemctl disable httpd

core/src/epicli/data/common/ansible/playbooks/roles/repository/tasks/teardown-Debian.yml → core/src/epicli/data/common/ansible/playbooks/roles/repository/tasks/Debian/teardown.yml

@@ -1,5 +1,4 @@
 ---
-
 - name: Enable system repositories and disable epirepo for offline mode
   block:
     - name: Enable system repositories
@@ -17,4 +16,4 @@
       shell: /tmp/{{ ansible_os_family }}/disable-epirepo-client-rh.sh
   when:
     - not groups['kubernetes_master'][0] == inventory_hostname
-    - offline_mode == false
+    - offline_mode == false

core/src/epicli/data/common/ansible/playbooks/roles/repository/tasks/RedHat/setup.yml

@@ -0,0 +1,19 @@
+---
+- name: Create epirepo repository
+  shell: >-
+    /tmp/epi-repository-setup-scripts/create-repository.sh /var/www/html/epirepo {{ offline_mode }} |&
+    tee /tmp/epi-repository-setup-scripts/create-repository.log
+  when:
+    - not custom_repository_url
+    - inventory_hostname in groups['repository']
+
+- name: Disable system repositories and set up epirepo
+  block:
+    - name: Create list of enabled repositories
+      shell: /tmp/epi-repository-setup-scripts/create-enabled-system-repos-list.sh
+    - name: Disable system repositories
+      shell: /tmp/epi-repository-setup-scripts/disable-system-repos.sh
+    - name: Add and enable epirepo
+      shell: >-
+        /tmp/epi-repository-setup-scripts/add-epirepo-client.sh {{ repository_url }} |&
+        tee /tmp/epi-repository-setup-scripts/add-epirepo-client.log

core/src/epicli/data/common/ansible/playbooks/roles/repository/tasks/RedHat/teardown.yml

@@ -0,0 +1,12 @@
+---
+- name: Re-enable system repositories
+  shell: /tmp/epi-repository-setup-scripts/enable-system-repos.sh
+
+- name: Disable epirepo
+  shell: /tmp/epi-repository-setup-scripts/disable-epirepo-client.sh
+
+- name: Disable epirepo server
+  shell: /tmp/epi-repository-setup-scripts/disable-repository-server.sh
+  when:
+    - not custom_repository_url
+    - inventory_hostname in groups['repository']

core/src/epicli/data/common/ansible/playbooks/roles/repository/tasks/download-requirements.yml

@@ -1,15 +1,31 @@
 ---
+# download-requirements-done.flag file is used to avoid re-downloading requirements (to save time)
+# this is to be optimized in the future
+
+- name: Check if flag file exists
+  stat:
+    path: /tmp/epi-download-requirements/download-requirements-done.flag
+  register: stat_result
+
+- name: Remove download-requirements-done.flag file if expired
+  file:
+    path: /tmp/epi-download-requirements/download-requirements-done.flag
+    state: absent
+  when:
+    - stat_result.stat.exists
+    - (ansible_date_time.epoch|int - stat_result.stat.mtime) > 3600 # older than 1h
+
 - name: Check whether to run download script
   stat:
     path: /tmp/epi-download-requirements/download-requirements-done.flag
   register: stat_result
 
 - name: Run download-requirements script, it can take a long time
   shell: /tmp/epi-download-requirements/download-requirements.sh /var/www/html/epirepo
-  when: stat_result.stat.exists == False
+  when: not stat_result.stat.exists
 
 - name: Create flag file to not re-download requirements next time
   file:
     path: /tmp/epi-download-requirements/download-requirements-done.flag
     state: touch
-  when: stat_result.stat.exists == False
+  when: not stat_result.stat.exists

core/src/epicli/data/common/ansible/playbooks/roles/repository/tasks/setup.yml

@@ -1,7 +1,7 @@
 ---
 - name: Set mode
   set_fact:
-    offline_mode: "{{ offline_requirements | length | bool }}"
+    offline_mode: "{{ offline_requirements | length > 0 }}"
 
 - name: Print mode
   debug:
@@ -23,7 +23,7 @@
 
     - name: Copy files for downloading requirements to repository host
       copy:
-        src: "prepare/{{ download_script_subdir }}/"
+        src: "download-requirements/{{ download_script_subdir }}/"
         dest: /tmp/epi-download-requirements
 
     - name: Copy skopeo binary to repository host
@@ -42,23 +42,33 @@
 
   when:
     - not offline_mode
-    - groups['kubernetes_master'][0] == inventory_hostname #TODO: dedicated host for repository in Ansible inventory
     - not custom_repository_url
+    - inventory_hostname in groups['repository']
 
-- name: Copy repository configuration scripts
+- name: Copy repository client configuration scripts
   copy:
-    src: "repository/{{ ansible_os_family }}/"
+    src: "client/{{ ansible_os_family }}/"
     dest: "/tmp/epi-repository-setup-scripts"
     mode: a+x
 
-- name: Copy requirements for offline installation (epi-repo) to repository host
+- name: Copy repository server configuration scripts
   copy:
-    src: /shared/epi-repo/ #TODO: This needs to be configurable
-    dest: /var/www/html/epi-repo
+    src: "server/{{ ansible_os_family }}/"
+    dest: "/tmp/epi-repository-setup-scripts"
+    mode: a+x
+  when:
+    - not offline_mode
+    - not custom_repository_url
+    - inventory_hostname in groups['repository']
+
+- name: Copy requirements for offline installation to repository host
+  copy:
+    src: "{{ offline_requirements }}/" #TODO: handle case with double slash
+    dest: /var/www/html/epirepo
   when:
     - offline_mode
-    - groups['kubernetes_master'][0] == inventory_hostname
     - not custom_repository_url
+    - inventory_hostname in groups['repository']
 
-- name: Configure repository and clients
-  include_tasks: "setup-{{ ansible_os_family }}.yml"
+- name: Set up repositories
+  include_tasks: "{{ ansible_os_family }}/setup.yml"

core/src/epicli/data/common/ansible/playbooks/roles/repository/tasks/teardown.yml

@@ -1,2 +1,2 @@
-- name: Configure repository and clients
-  include_tasks: "teardown-{{ ansible_os_family }}.yml"
+- name: Re-enable system repositories and disable epirepo
+  include_tasks: "{{ ansible_os_family }}/teardown.yml"

core/src/epicli/data/common/defaults/configuration/feature-mapping.yml

@@ -4,6 +4,8 @@ title: "Feature mapping to roles"
 name: default
 specification:
   available_roles:
+    - name: repository
+      enabled: yes
     - name: kubernetes-master
       enabled: yes
     - name: kubernetes-node
@@ -72,6 +74,7 @@ specification:
       - node-exporter
       - filebeat
     custom:
+      - repository
       - kubernetes-master
       - node-exporter
       - filebeat
@@ -84,12 +87,14 @@ specification:
       - elasticsearch
       - elasticsearch-curator
     single_machine:
+      - repository
       - kubernetes-master
       - applications
       - rabbitmql
       - postgresql
     kubernetes_master:
       - kubernetes-master
+      - repository
       - applications
       - node-exporter
       - filebeat