Offline mode progress (#555)

* yum makecache -> yum makecache fast #536

* Added versions for docker-ce & docker-ce-cli #536

* Added property 'download_done_flag_expire_minutes' #536

* enabled-system-repos.txt in /var/tmp #536

* Clean up temporary files #536

* Added execution time #536

core/src/epicli/data/common/ansible/playbooks/repository.yml

@@ -0,0 +1,5 @@
+---
+# This playbook is empty by purpose, just to enable repository role in configuration/feature-mapping
+# to populate defaults/configuration to Ansible vars
+- hosts: []
+  tasks: []

core/src/epicli/data/common/ansible/playbooks/roles/docker/tasks/main.yml

@@ -9,8 +9,8 @@
     - apt-transport-https
     - ca-certificates
     - containerd.io
-    - docker-ce-cli
-    - docker-ce
+    - docker-ce-cli=5:18.09.*
+    - docker-ce=5:18.09.*
   when: ansible_os_family == "Debian"
 
 - name: Install Docker for RedHat family
@@ -19,7 +19,8 @@
     state: present
   loop:
       - container-selinux
-      - docker-ce-18.09.9
+      - docker-ce-cli-18.09.*
+      - docker-ce-18.09.*
   when: ansible_os_family == "RedHat"
 
 - include_tasks: configure-docker.yml

core/src/epicli/data/common/ansible/playbooks/roles/repository/files/client/RedHat/add-epirepo-client.sh

@@ -13,5 +13,5 @@ gpgcheck=0
 EOF
 
 yum-config-manager --enable epirepo
-yum makecache
-yum repolist
+yum makecache fast
+yum repolist

core/src/epicli/data/common/ansible/playbooks/roles/repository/files/client/RedHat/create-enabled-system-repos-list.sh

@@ -1,6 +1,6 @@
 #!/bin/bash -eu
 
-ENABLED_REPOS_FILE=/tmp/enabled-system-repos.txt
+ENABLED_REPOS_FILE=/var/tmp/enabled-system-repos.txt
 
 if [ ! -f "$ENABLED_REPOS_FILE" ]; then
   yum repolist -v enabled | grep -i Repo-id | awk -F ":" '{print $2}' |  sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' | awk -F "/" '{print $1}' > $ENABLED_REPOS_FILE

core/src/epicli/data/common/ansible/playbooks/roles/repository/files/client/RedHat/disable-epirepo-client.sh

@@ -1,5 +1,5 @@
 #!/bin/bash -eu
 
 yum-config-manager --disable epirepo
-yum makecache
-yum repolist
+yum clean all --disablerepo='*' --enablerepo=epirepo
+yum repolist

core/src/epicli/data/common/ansible/playbooks/roles/repository/files/client/RedHat/disable-system-repos.sh

@@ -1,10 +1,9 @@
 #!/bin/bash -eu
 
-REPOS_LIST_FILE=/tmp/enabled-system-repos.txt
+REPOS_LIST_FILE=/var/tmp/enabled-system-repos.txt
 
 cat $REPOS_LIST_FILE | while read repository
 do
   echo "Disabling repository: $repository"
   yum-config-manager --disable $repository
-done
-
+done

core/src/epicli/data/common/ansible/playbooks/roles/repository/files/client/RedHat/enable-system-repos.sh

@@ -1,9 +1,9 @@
 #!/bin/bash -eu
 
-REPOS_LIST_FILE=/tmp/enabled-system-repos.txt
+REPOS_LIST_FILE=/var/tmp/enabled-system-repos.txt
 
 cat $REPOS_LIST_FILE | while read repository
 do
   echo "Enabling repository: $repository"
   yum-config-manager --enable $repository
-done
+done

core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/centos-7/download-requirements.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-# VERSION 1.2.1
+# VERSION 1.2.3
 
 set -euo pipefail
 
@@ -111,9 +111,7 @@ download_packages() {
 
 echol() {
 	echo -e "$@"
-	if [[ $LOG_TO_JOURNAL == 'YES' ]]; then
-		echo -e "$@" | systemd-cat --identifier=$SCRIPT_FILE_NAME
-	else # log to $LOG_FILE_PATH
+	if [[ $CREATE_LOGFILE == 'yes' ]]; then
 		local timestamp=$(date +"%b %e %H:%M:%S")
 		echo -e "${timestamp}: $@" >> "$LOG_FILE_PATH"
 	fi
@@ -304,15 +302,16 @@ usage() {
 # === Start ===
 
 [ $# -gt 0 ] || usage 1 >&2
+readonly START_TIME=$(date +%s)
 
 # --- Parse arguments ---
 
 POSITIONAL_ARGS=()
-LOG_TO_JOURNAL='NO'
+CREATE_LOGFILE='yes'
 while [[ $# -gt 0 ]]; do
 case $1 in
-	--log-to-journal)
-	LOG_TO_JOURNAL='YES'
+	--no-logfile)
+	CREATE_LOGFILE='no'
 	shift # past argument
 	;;
 	*) # unknown option
@@ -481,7 +480,7 @@ if ! is_package_installed 'epel-release'; then
 	install_package 'https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm' 'epel-release'
 fi
 
-echol "Executing: yum -y makecache" && yum -y makecache
+echol "Executing: yum -y makecache fast" && yum -y makecache fast
 
 # --- Download packages ---
 
@@ -578,4 +577,6 @@ done
 # --- Clean up packages ---
 remove_installed_packages "$INSTALLED_PACKAGES_FILE_PATH"
 
-echol "$(basename $0) finished"
+readonly END_TIME=$(date +%s)
+
+echol "$(basename $0) finished, execution time: $(date -u -d @$((END_TIME-START_TIME)) +'%Hh:%Mm:%Ss')"

core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/download-requirements.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-# VERSION 1.2.1
+# VERSION 1.2.3
 
 set -euo pipefail
 
@@ -111,9 +111,7 @@ download_packages() {
 
 echol() {
 	echo -e "$@"
-	if [[ $LOG_TO_JOURNAL == 'YES' ]]; then
-		echo -e "$@" | systemd-cat --identifier=$SCRIPT_FILE_NAME
-	else # log to $LOG_FILE_PATH
+	if [[ $CREATE_LOGFILE == 'yes' ]]; then
 		local timestamp=$(date +"%b %e %H:%M:%S")
 		echo -e "${timestamp}: $@" >> "$LOG_FILE_PATH"
 	fi
@@ -321,15 +319,16 @@ usage() {
 # === Start ===
 
 [ $# -gt 0 ] || usage 1 >&2
+readonly START_TIME=$(date +%s)
 
 # --- Parse arguments ---
 
 POSITIONAL_ARGS=()
-LOG_TO_JOURNAL='NO'
+CREATE_LOGFILE='yes'
 while [[ $# -gt 0 ]]; do
 case $1 in
-	--log-to-journal)
-	LOG_TO_JOURNAL='YES'
+	--no-logfile)
+	CREATE_LOGFILE='no'
 	shift # past argument
 	;;
 	*) # unknown option
@@ -504,7 +503,7 @@ if ! is_package_installed 'epel-release'; then
 	install_package 'https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm' 'epel-release'
 fi
 
-echol "Executing: yum -y makecache" && yum -y makecache
+echol "Executing: yum -y makecache fast" && yum -y makecache fast
 
 # --- Download packages ---
 
@@ -601,4 +600,6 @@ done
 # --- Clean up packages ---
 remove_installed_packages "$INSTALLED_PACKAGES_FILE_PATH"
 
-echol "$(basename $0) finished"
+readonly END_TIME=$(date +%s)
+
+echol "$(basename $0) finished, execution time: $(date -u -d @$((END_TIME-START_TIME)) +'%Hh:%Mm:%Ss')"

core/src/epicli/data/common/ansible/playbooks/roles/repository/tasks/download-requirements.yml

@@ -13,7 +13,7 @@
     state: absent
   when:
     - stat_result.stat.exists
-    - (ansible_date_time.epoch|int - stat_result.stat.mtime) > 3600 # older than 1h
+    - (ansible_date_time.epoch|int - stat_result.stat.mtime) > (60 * specification.download_done_flag_expire_minutes)
 
 - name: Check whether to run download script
   stat:
@@ -23,7 +23,11 @@
 - name: |-
     Run download-requirements script, it can take a long time
     You can check progress on repository host with: journalctl -f -t download-requirements.sh
-  shell: /tmp/epi-download-requirements/download-requirements.sh /var/www/html/epirepo --log-to-journal
+  shell: >-
+    set -o pipefail && /tmp/epi-download-requirements/download-requirements.sh /var/www/html/epirepo --no-logfile |&
+    tee >(systemd-cat --identifier=download-requirements.sh)
+  args:
+    executable: /bin/bash
   when: not stat_result.stat.exists
 
 - name: Create flag file to not re-download requirements next time

core/src/epicli/data/common/ansible/playbooks/roles/repository/tasks/teardown.yml

@@ -1,2 +1,13 @@
 - name: Re-enable system repositories and disable epirepo
-  include_tasks: "{{ ansible_os_family }}/teardown.yml"
+  include_tasks: "{{ ansible_os_family }}/teardown.yml"
+
+- name: Clean up temporary files
+  file:
+    state: absent
+    path: "{{ path }}"
+  loop:
+    - /tmp/epi-download-requirements
+    - /tmp/epi-repository-setup-scripts
+    - /var/tmp/enabled-system-repos.txt
+  loop_control:
+    index_var: path

core/src/epicli/data/common/defaults/configuration/feature-mapping.yml

@@ -5,7 +5,7 @@ name: default
 specification:
   available_roles:
     - name: repository
-      enabled: no # disabled by purpose, there is no ansible/repository.yml
+      enabled: yes
     - name: kubernetes-master
       enabled: yes
     - name: kubernetes-node

core/src/epicli/data/common/defaults/configuration/repository.yml

@@ -3,4 +3,5 @@ version: 0.4.0
 title: "Epiphany requirements repository"
 name: default
 specification:
-  description: "Local repository of binaries required to install Epiphany"
+  description: "Local repository of binaries required to install Epiphany"
+  download_done_flag_expire_minutes: 120