From 3f6f0599b3bd2cb79b6ac6cd0c47b78b49c2ed76 Mon Sep 17 00:00:00 2001
From: to-bar <46519524+to-bar@users.noreply.github.com>
Date: Mon, 7 Oct 2019 16:41:20 +0200
Subject: [PATCH] Offline mode progress (#555)

* yum makecache -> yum makecache fast #536

* Added versions for docker-ce & docker-ce-cli #536

* Added property 'download_done_flag_expire_minutes' #536

* enabled-system-repos.txt in /var/tmp #536

* Clean up temporary files #536

* Added execution time #536
---
 .../common/ansible/playbooks/repository.yml   |  5 +++++
 .../playbooks/roles/docker/tasks/main.yml     |  7 ++++---
 .../files/client/RedHat/add-epirepo-client.sh |  4 ++--
 .../create-enabled-system-repos-list.sh       |  2 +-
 .../client/RedHat/disable-epirepo-client.sh   |  4 ++--
 .../client/RedHat/disable-system-repos.sh     |  5 ++---
 .../client/RedHat/enable-system-repos.sh      |  4 ++--
 .../centos-7/download-requirements.sh         | 19 ++++++++++---------
 .../redhat-7/download-requirements.sh         | 19 ++++++++++---------
 .../tasks/download-requirements.yml           |  8 ++++++--
 .../roles/repository/tasks/teardown.yml       | 13 ++++++++++++-
 .../configuration/feature-mapping.yml         |  2 +-
 .../defaults/configuration/repository.yml     |  3 ++-
 13 files changed, 59 insertions(+), 36 deletions(-)
 create mode 100644 core/src/epicli/data/common/ansible/playbooks/repository.yml

diff --git a/core/src/epicli/data/common/ansible/playbooks/repository.yml b/core/src/epicli/data/common/ansible/playbooks/repository.yml
new file mode 100644
index 0000000000..3fcc144ef1
--- /dev/null
+++ b/core/src/epicli/data/common/ansible/playbooks/repository.yml
@@ -0,0 +1,5 @@
+---
+# This playbook is empty by purpose, just to enable repository role in configuration/feature-mapping
+# to populate defaults/configuration to Ansible vars
+- hosts: []
+  tasks: []
\ No newline at end of file
diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/docker/tasks/main.yml b/core/src/epicli/data/common/ansible/playbooks/roles/docker/tasks/main.yml
index d706438955..e71d327dd1 100644
--- a/core/src/epicli/data/common/ansible/playbooks/roles/docker/tasks/main.yml
+++ b/core/src/epicli/data/common/ansible/playbooks/roles/docker/tasks/main.yml
@@ -9,8 +9,8 @@
     - apt-transport-https
     - ca-certificates
     - containerd.io
-    - docker-ce-cli
-    - docker-ce
+    - docker-ce-cli=5:18.09.*
+    - docker-ce=5:18.09.*
   when: ansible_os_family == "Debian"
 
 - name: Install Docker for RedHat family
@@ -19,7 +19,8 @@
     state: present
   loop:
       - container-selinux
-      - docker-ce-18.09.9
+      - docker-ce-cli-18.09.*
+      - docker-ce-18.09.*
   when: ansible_os_family == "RedHat"
 
 - include_tasks: configure-docker.yml
diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/client/RedHat/add-epirepo-client.sh b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/client/RedHat/add-epirepo-client.sh
index fd7781d5cf..035e148361 100644
--- a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/client/RedHat/add-epirepo-client.sh
+++ b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/client/RedHat/add-epirepo-client.sh
@@ -13,5 +13,5 @@ gpgcheck=0
 EOF
 
 yum-config-manager --enable epirepo
-yum makecache
-yum repolist
+yum makecache fast
+yum repolist
\ No newline at end of file
diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/client/RedHat/create-enabled-system-repos-list.sh b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/client/RedHat/create-enabled-system-repos-list.sh
index e3703950b1..12416501aa 100644
--- a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/client/RedHat/create-enabled-system-repos-list.sh
+++ b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/client/RedHat/create-enabled-system-repos-list.sh
@@ -1,6 +1,6 @@
 #!/bin/bash -eu
 
-ENABLED_REPOS_FILE=/tmp/enabled-system-repos.txt
+ENABLED_REPOS_FILE=/var/tmp/enabled-system-repos.txt
 
 if [ ! -f "$ENABLED_REPOS_FILE" ]; then
   yum repolist -v enabled | grep -i Repo-id | awk -F ":" '{print $2}' |  sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' | awk -F "/" '{print $1}' > $ENABLED_REPOS_FILE
diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/client/RedHat/disable-epirepo-client.sh b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/client/RedHat/disable-epirepo-client.sh
index 612e019d5e..24124a975a 100644
--- a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/client/RedHat/disable-epirepo-client.sh
+++ b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/client/RedHat/disable-epirepo-client.sh
@@ -1,5 +1,5 @@
 #!/bin/bash -eu
 
 yum-config-manager --disable epirepo
-yum makecache
-yum repolist
+yum clean all --disablerepo='*' --enablerepo=epirepo
+yum repolist
\ No newline at end of file
diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/client/RedHat/disable-system-repos.sh b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/client/RedHat/disable-system-repos.sh
index 51a4f0deee..858c21d8f5 100644
--- a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/client/RedHat/disable-system-repos.sh
+++ b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/client/RedHat/disable-system-repos.sh
@@ -1,10 +1,9 @@
 #!/bin/bash -eu
 
-REPOS_LIST_FILE=/tmp/enabled-system-repos.txt
+REPOS_LIST_FILE=/var/tmp/enabled-system-repos.txt
 
 cat $REPOS_LIST_FILE | while read repository
 do
   echo "Disabling repository: $repository"
   yum-config-manager --disable $repository
-done
-
+done
\ No newline at end of file
diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/client/RedHat/enable-system-repos.sh b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/client/RedHat/enable-system-repos.sh
index 77768c2bae..42aeef3d73 100644
--- a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/client/RedHat/enable-system-repos.sh
+++ b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/client/RedHat/enable-system-repos.sh
@@ -1,9 +1,9 @@
 #!/bin/bash -eu
 
-REPOS_LIST_FILE=/tmp/enabled-system-repos.txt
+REPOS_LIST_FILE=/var/tmp/enabled-system-repos.txt
 
 cat $REPOS_LIST_FILE | while read repository
 do
   echo "Enabling repository: $repository"
   yum-config-manager --enable $repository
-done
+done
\ No newline at end of file
diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/centos-7/download-requirements.sh b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/centos-7/download-requirements.sh
index e0c862b967..8864fb15d7 100644
--- a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/centos-7/download-requirements.sh
+++ b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/centos-7/download-requirements.sh
@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-# VERSION 1.2.1
+# VERSION 1.2.3
 
 set -euo pipefail
 
@@ -111,9 +111,7 @@ download_packages() {
 
 echol() {
 	echo -e "$@"
-	if [[ $LOG_TO_JOURNAL == 'YES' ]]; then
-		echo -e "$@" | systemd-cat --identifier=$SCRIPT_FILE_NAME
-	else # log to $LOG_FILE_PATH
+	if [[ $CREATE_LOGFILE == 'yes' ]]; then
 		local timestamp=$(date +"%b %e %H:%M:%S")
 		echo -e "${timestamp}: $@" >> "$LOG_FILE_PATH"
 	fi
@@ -304,15 +302,16 @@ usage() {
 # === Start ===
 
 [ $# -gt 0 ] || usage 1 >&2
+readonly START_TIME=$(date +%s)
 
 # --- Parse arguments ---
 
 POSITIONAL_ARGS=()
-LOG_TO_JOURNAL='NO'
+CREATE_LOGFILE='yes'
 while [[ $# -gt 0 ]]; do
 case $1 in
-	--log-to-journal)
-	LOG_TO_JOURNAL='YES'
+	--no-logfile)
+	CREATE_LOGFILE='no'
 	shift # past argument
 	;;
 	*) # unknown option
@@ -481,7 +480,7 @@ if ! is_package_installed 'epel-release'; then
 	install_package 'https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm' 'epel-release'
 fi
 
-echol "Executing: yum -y makecache" && yum -y makecache
+echol "Executing: yum -y makecache fast" && yum -y makecache fast
 
 # --- Download packages ---
 
@@ -578,4 +577,6 @@ done
 # --- Clean up packages ---
 remove_installed_packages "$INSTALLED_PACKAGES_FILE_PATH"
 
-echol "$(basename $0) finished"
\ No newline at end of file
+readonly END_TIME=$(date +%s)
+
+echol "$(basename $0) finished, execution time: $(date -u -d @$((END_TIME-START_TIME)) +'%Hh:%Mm:%Ss')"
\ No newline at end of file
diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/download-requirements.sh b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/download-requirements.sh
index 20d77982f4..0b5761f4e3 100644
--- a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/download-requirements.sh
+++ b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/download-requirements.sh
@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-# VERSION 1.2.1
+# VERSION 1.2.3
 
 set -euo pipefail
 
@@ -111,9 +111,7 @@ download_packages() {
 
 echol() {
 	echo -e "$@"
-	if [[ $LOG_TO_JOURNAL == 'YES' ]]; then
-		echo -e "$@" | systemd-cat --identifier=$SCRIPT_FILE_NAME
-	else # log to $LOG_FILE_PATH
+	if [[ $CREATE_LOGFILE == 'yes' ]]; then
 		local timestamp=$(date +"%b %e %H:%M:%S")
 		echo -e "${timestamp}: $@" >> "$LOG_FILE_PATH"
 	fi
@@ -321,15 +319,16 @@ usage() {
 # === Start ===
 
 [ $# -gt 0 ] || usage 1 >&2
+readonly START_TIME=$(date +%s)
 
 # --- Parse arguments ---
 
 POSITIONAL_ARGS=()
-LOG_TO_JOURNAL='NO'
+CREATE_LOGFILE='yes'
 while [[ $# -gt 0 ]]; do
 case $1 in
-	--log-to-journal)
-	LOG_TO_JOURNAL='YES'
+	--no-logfile)
+	CREATE_LOGFILE='no'
 	shift # past argument
 	;;
 	*) # unknown option
@@ -504,7 +503,7 @@ if ! is_package_installed 'epel-release'; then
 	install_package 'https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm' 'epel-release'
 fi
 
-echol "Executing: yum -y makecache" && yum -y makecache
+echol "Executing: yum -y makecache fast" && yum -y makecache fast
 
 # --- Download packages ---
 
@@ -601,4 +600,6 @@ done
 # --- Clean up packages ---
 remove_installed_packages "$INSTALLED_PACKAGES_FILE_PATH"
 
-echol "$(basename $0) finished"
\ No newline at end of file
+readonly END_TIME=$(date +%s)
+
+echol "$(basename $0) finished, execution time: $(date -u -d @$((END_TIME-START_TIME)) +'%Hh:%Mm:%Ss')"
\ No newline at end of file
diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/repository/tasks/download-requirements.yml b/core/src/epicli/data/common/ansible/playbooks/roles/repository/tasks/download-requirements.yml
index a2f8e032c9..fd9f300a7d 100644
--- a/core/src/epicli/data/common/ansible/playbooks/roles/repository/tasks/download-requirements.yml
+++ b/core/src/epicli/data/common/ansible/playbooks/roles/repository/tasks/download-requirements.yml
@@ -13,7 +13,7 @@
     state: absent
   when:
     - stat_result.stat.exists
-    - (ansible_date_time.epoch|int - stat_result.stat.mtime) > 3600 # older than 1h
+    - (ansible_date_time.epoch|int - stat_result.stat.mtime) > (60 * specification.download_done_flag_expire_minutes)
 
 - name: Check whether to run download script
   stat:
@@ -23,7 +23,11 @@
 - name: |-
     Run download-requirements script, it can take a long time
     You can check progress on repository host with: journalctl -f -t download-requirements.sh
-  shell: /tmp/epi-download-requirements/download-requirements.sh /var/www/html/epirepo --log-to-journal
+  shell: >-
+    set -o pipefail && /tmp/epi-download-requirements/download-requirements.sh /var/www/html/epirepo --no-logfile |&
+    tee >(systemd-cat --identifier=download-requirements.sh)
+  args:
+    executable: /bin/bash
   when: not stat_result.stat.exists
 
 - name: Create flag file to not re-download requirements next time
diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/repository/tasks/teardown.yml b/core/src/epicli/data/common/ansible/playbooks/roles/repository/tasks/teardown.yml
index 397176776a..287cca0b6c 100644
--- a/core/src/epicli/data/common/ansible/playbooks/roles/repository/tasks/teardown.yml
+++ b/core/src/epicli/data/common/ansible/playbooks/roles/repository/tasks/teardown.yml
@@ -1,2 +1,13 @@
 - name: Re-enable system repositories and disable epirepo
-  include_tasks: "{{ ansible_os_family }}/teardown.yml"
\ No newline at end of file
+  include_tasks: "{{ ansible_os_family }}/teardown.yml"
+
+- name: Clean up temporary files
+  file:
+    state: absent
+    path: "{{ path }}"
+  loop:
+    - /tmp/epi-download-requirements
+    - /tmp/epi-repository-setup-scripts
+    - /var/tmp/enabled-system-repos.txt
+  loop_control:
+    index_var: path
\ No newline at end of file
diff --git a/core/src/epicli/data/common/defaults/configuration/feature-mapping.yml b/core/src/epicli/data/common/defaults/configuration/feature-mapping.yml
index 1c913df5f2..436bfb070a 100644
--- a/core/src/epicli/data/common/defaults/configuration/feature-mapping.yml
+++ b/core/src/epicli/data/common/defaults/configuration/feature-mapping.yml
@@ -5,7 +5,7 @@ name: default
 specification:
   available_roles:
     - name: repository
-      enabled: no # disabled by purpose, there is no ansible/repository.yml
+      enabled: yes
     - name: kubernetes-master
       enabled: yes
     - name: kubernetes-node
diff --git a/core/src/epicli/data/common/defaults/configuration/repository.yml b/core/src/epicli/data/common/defaults/configuration/repository.yml
index f69d916fcf..2cbd8b8a91 100644
--- a/core/src/epicli/data/common/defaults/configuration/repository.yml
+++ b/core/src/epicli/data/common/defaults/configuration/repository.yml
@@ -3,4 +3,5 @@ version: 0.4.0
 title: "Epiphany requirements repository"
 name: default
 specification:
-  description: "Local repository of binaries required to install Epiphany"
\ No newline at end of file
+  description: "Local repository of binaries required to install Epiphany"
+  download_done_flag_expire_minutes: 120
\ No newline at end of file