Merge pull request #517 from erzetpe/feature/offline-mode

Add scripts and repository role for setup and teardown

core/src/epicli/cli/engine/ansible/AnsibleRunner.py

@@ -47,11 +47,20 @@ def run(self):
 
         self.ansible_vars_generator.run()
 
+        repository_setup_play_result = self.ansible_command.run_playbook_with_retries(inventory=inventory_path,
+                                                                            playbook_path=os.path.join(
+                                                                                get_ansible_path(
+                                                                                    self.cluster_model.specification.name),
+                                                                                "repository-setup.yml"), retries=5)
+
+        if repository_setup_play_result != 0:
+            return
+
         common_play_result = self.ansible_command.run_playbook_with_retries(inventory=inventory_path,
                                                                             playbook_path=os.path.join(
                                                                                 get_ansible_path(
                                                                                     self.cluster_model.specification.name),
-                                                                                "common.yml"), retries=5)
+                                                                                "common.yml"), retries=1)
         if common_play_result != 0:
             return
 
@@ -65,3 +74,12 @@ def run(self):
                                                                              to_role_name(role) + ".yml"), retries=1)
             if play_result != 0:
                 break
+
+        repository_teardown_play_result = self.ansible_command.run_playbook_with_retries(inventory=inventory_path,
+                                                                            playbook_path=os.path.join(
+                                                                                get_ansible_path(
+                                                                                    self.cluster_model.specification.name),
+                                                                                "repository-teardown.yml"), retries=1)
+
+        if repository_teardown_play_result != 0:
+            return

core/src/epicli/data/aws/defaults/infrastructure/virtual-machine.yml

@@ -147,7 +147,7 @@ specification:
        source_port_range: "*"
        destination_port_range: "0"
        source_address_prefix: "0.0.0.0/0"
-       destination_address_prefix: "0.0.0.0/0"       
+       destination_address_prefix: "0.0.0.0/0"
 ---
 kind: infrastructure/virtual-machine
 version: 0.3.0
@@ -223,6 +223,16 @@ specification:
        destination_port_range: "22"
        source_address_prefix: "0.0.0.0/0"
        destination_address_prefix: "0.0.0.0/0"
+     - name: repository
+       description: Allow repository traffic
+       priority: 302
+       direction: Inbound
+       access: Allow
+       protocol: Tcp
+       source_port_range: "*"
+       destination_port_range: "80"
+       source_address_prefix: "10.1.0.0/20"
+       destination_address_prefix: "0.0.0.0/0"
      - name: node_exporter
        description: Allow node_exporter traffic
        priority: 302

core/src/epicli/data/common/ansible/playbooks/repository-setup.yml

@@ -0,0 +1,12 @@
+---
+# Ansible playbook for disabling/enabling repositories before/after Epiphany installation
+
+- hosts: all
+  pre_tasks:
+    - name: Set mode to setup
+      set_fact:
+        repository_mode: setup
+  become: true
+  become_method: sudo
+  roles:
+    - repository

core/src/epicli/data/common/ansible/playbooks/repository-teardown.yml

@@ -0,0 +1,12 @@
+---
+# Ansible playbook for disabling/enabling repositories before/after Epiphany installation
+
+- hosts: all
+  pre_tasks:
+    - name: Set mode to teardown
+      set_fact:
+        repository_mode: teardown
+  become: true
+  become_method: sudo
+  roles:
+    - repository

core/src/epicli/data/common/ansible/playbooks/roles/repository/files/Debian/scripts/create-repository-deb.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+PACKAGE_LIST=$(cat /root/deb-package-list.txt)
+DOWNLOAD_DIRECTORY=/root/packages
+LOG_FILE=/root/script-execution.log
+
+WWW_SERVER_PATH=/var/www/html;
+
+REPOSITORY_PATH=$WWW_SERVER_PATH/repos;
+FILES_PATH=$WWW_SERVER_PATH/files;
+IMAGES_PATH=$WWW_SERVER_PATH/images;
+
+apt install -y apache2 reprepro;
+systemctl start apache2
+apt clean;
+
+
+mkdir -p $REPOSITORY_PATH;
+mkdir -p $REPOSITORY_PATH/conf;
+
+cat << EOF > $REPOSITORY_PATH/conf/distributions 
+Origin: epiphany.offline.repo
+Label: epiphany.offline.repo
+Codename: bionic
+Architectures: i386 amd64
+Components: main restricted universe multiverse
+Description: Epiphany Offline Repository
+EOF
+
+for package in $PACKAGE_LIST ; do
+	echo "$package:" | tee $LOG_FILE;
+	apt-get install -y  --download-only $package | tee $LOG_FILE ;
+done
+
+reprepro --basedir $REPOSITORY_PATH includedeb bionic /var/cache/apt/archives/*.deb;
+

core/src/epicli/data/common/ansible/playbooks/roles/repository/files/Debian/scripts/setup-repo-client-deb.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+SERVER_IP=$1;
+DATE=`date +%Y.%m.%d-%H.%M.%S`;
+
+curl -I -L $SERVER_IP/repos | grep "HTTP/1.1 200 OK";
+
+cp /etc/apt/sources.list /etc/apt/sources.list.bak_$DATE;
+echo "deb [trusted=yes] http://$SERVER_IP/repos/ bionic main" > /etc/apt/sources.list;
+
+apt-cache policy;
+
+apt update;
+

core/src/epicli/data/common/ansible/playbooks/roles/repository/files/RedHat/rh-package-list.txt

@@ -0,0 +1,22 @@
+libselinux-python
+libsemanage-python
+firewalld
+bash-completion
+ca-certificates
+net-tools
+tar
+nmap-ncat
+curl
+tmux
+fping
+iftop
+htop
+vim-enhanced
+sysstat
+python-setuptools
+openssl
+yum-plugin-versionlock
+logrotate
+ebtables
+ethtool
+telnet

core/src/epicli/data/common/ansible/playbooks/roles/repository/files/RedHat/scripts/create-repository-rh.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+
+PACKAGE_LIST=$(cat $1)
+LOG_FILE=/root/script-execution.log
+
+WWW_SERVER_PATH=/var/www/html;
+
+REPOSITORY_PATH=$WWW_SERVER_PATH/repos;
+FILES_PATH=$WWW_SERVER_PATH/files;
+IMAGES_PATH=$WWW_SERVER_PATH/images;
+
+mkdir -p $WWW_SERVER_PATH;
+mkdir -p $REPOSITORY_PATH;
+mkdir -p $FILES_PATH;
+mkdir -p $IMAGES_PATH;
+
+yum install -y httpd createrepo  yum-utils;
+
+for package in $PACKAGE_LIST ; do
+        echo "========== $package =========" | tee $LOG_FILE;
+        repoquery -a --qf  '%{ui_nevra}' $package;
+        repoquery -a --qf  '%{ui_nevra}' $package | xargs yumdownloader --destdir $REPOSITORY_PATH | tee $LOG_FILE;
+        echo "========== $package - dependencies =========" | tee $LOG_FILE;
+        repoquery -R --resolve -a --qf  '%{ui_nevra}' $package;
+        repoquery -R --resolve -a --qf  '%{ui_nevra}' $package | xargs yumdownloader --destdir $REPOSITORY_PATH | tee $LOG_FILE;
+done
+
+setenforce 0;
+systemctl start httpd;
+
+createrepo $REPOSITORY_PATH;
+
+

core/src/epicli/data/common/ansible/playbooks/roles/repository/files/RedHat/scripts/disable-epirepo-client-rh.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+yum-config-manager --disable epirepo*;
+yum makecache;
+yum repolist;
+

core/src/epicli/data/common/ansible/playbooks/roles/repository/files/RedHat/scripts/disable-repository-rh.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+systemctl stop httpd;
+systemctl disable httpd;

core/src/epicli/data/common/ansible/playbooks/roles/repository/files/RedHat/scripts/disable-system-repos.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+REPOS_LIST_FILE=/tmp/enabled-system-repos.txt
+
+cat $REPOS_LIST_FILE | while read line
+do
+  echo $line;
+  yum-config-manager --disable $line;
+done
+

core/src/epicli/data/common/ansible/playbooks/roles/repository/files/RedHat/scripts/enable-system-repos.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+REPOS_LIST_FILE=/tmp/enabled-system-repos.txt
+
+cat $REPOS_LIST_FILE | while read line
+do
+  echo $line;
+  yum-config-manager --enable $line;
+done
+

core/src/epicli/data/common/ansible/playbooks/roles/repository/files/RedHat/scripts/generate-enabled-system-repository-list.sh

@@ -0,0 +1,4 @@
+ENABLED_REPOS_FILE=/tmp/enabled-system-repos.txt
+if [ test ! -f "$ENABLED_REPOS_FILE" ]; then
+    yum repolist -v enabled | grep -i Repo-id | awk -F ":" '{print $2}' |  sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' | awk -F "/" '{print $1}' > $ENABLED_REPOS_FILE;
+fi

core/src/epicli/data/common/ansible/playbooks/roles/repository/files/RedHat/scripts/setup-epirepo-client-rh.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+SERVER_IP=$1;
+
+curl -I -L $SERVER_IP/repos | grep "HTTP/1.1 200 OK";
+
+sed -i -e "s/enabled=1/enabled=0/g" /etc/yum.repos.d/*.repo;
+
+cat << EOF > /etc/yum.repos.d/epirepo.repo
+[epirepo]
+name=epirepo
+baseurl=http://$SERVER_IP/repos/
+enabled=1
+gpgcheck=0
+EOF
+
+yum-config-manager --enable epirepo*;
+yum makecache;
+yum repolist;
+

core/src/epicli/data/common/ansible/playbooks/roles/repository/tasks/main.yml

@@ -0,0 +1,15 @@
+---
+
+- name: Copy data files
+  copy:
+    src: "{{ ansible_os_family }}/"
+    dest: "/tmp/{{ ansible_os_family }}"
+
+- name: Copy repository configuration scripts
+  copy:
+    src: "{{ ansible_os_family }}/scripts/"
+    dest: "/tmp/{{ ansible_os_family }}"
+    mode: a+x
+
+- name: Configure repository and clients RedHat
+  include_tasks: "{{ repository_mode }}-{{ ansible_os_family }}.yml"

core/src/epicli/data/common/ansible/playbooks/roles/repository/tasks/setup-RedHat.yml

@@ -0,0 +1,32 @@
+---
+
+- name: Copy data files
+  copy:
+    src: "{{ ansible_os_family }}/"
+    dest: "/tmp/{{ ansible_os_family }}"
+
+- name: Copy repository configuration scripts
+  copy:
+    src: "{{ ansible_os_family }}/scripts/"
+    dest: "/tmp/{{ ansible_os_family }}"
+    mode: a+x
+
+- name: Download packages and create repository
+  shell: /tmp/{{ ansible_os_family }}/create-repository-rh.sh /tmp/{{ ansible_os_family }}/rh-package-list.txt
+  when:
+      - groups['kubernetes_master'][0] == inventory_hostname
+
+- name: Create active repositories list
+  shell: /tmp/{{ ansible_os_family }}/generate-enabled-system-repository-list.sh
+  when:
+      - not groups['kubernetes_master'][0] == inventory_hostname
+
+- name: Disable active system repositories
+  shell: /tmp/{{ ansible_os_family }}/disable-system-repos.sh
+  when:
+      - not groups['kubernetes_master'][0] == inventory_hostname
+
+- name: Setup epirepo on clients
+  shell: /tmp/{{ ansible_os_family }}/setup-epirepo-client-rh.sh {{ groups['kubernetes_master'][0] }}
+  when:
+      - not groups['kubernetes_master'][0] == inventory_hostname

core/src/epicli/data/common/ansible/playbooks/roles/repository/tasks/teardown-RedHat.yml

@@ -0,0 +1,11 @@
+---
+
+- name: Enable system repositories
+  shell: /tmp/{{ ansible_os_family }}/enable-system-repos.sh
+  when:
+      - not groups['kubernetes_master'][0] == inventory_hostname
+
+- name: Disable epirepo on clients
+  shell: /tmp/{{ ansible_os_family }}/disable-epirepo-client-rh.sh
+  when:
+      - not groups['kubernetes_master'][0] == inventory_hostname