Offline mode - progress for RedHat (hitachienergy#552)

* Added --log-to-journal option hitachienergy#536 * download-requirements script logs to journal hitachienergy#536 * Added python-slip-dbus for firewalld on AWS hitachienergy#536 * Disabled repository role in feature mappings hitachienergy#536 * Static version for docker-ce hitachienergy#536 * download-reqs: yum makecache fast -> yum makecache hitachienergy#536 * Do not install epel-release hitachienergy#536 * Download latest versions of dependencies hitachienergy#536 * Optimizations and better naming hitachienergy#536 * Colons -> hyphens in file names of images hitachienergy#536 * jmx_prometheus_javaagent v0.12.0 hitachienergy#536 * Better task names in zookeeper role
to-bar · Oct 5, 2019 · 1f1f8c4 · 1f1f8c4
1 parent 8c6f254
commit 1f1f8c4
Show file tree

Hide file tree

Showing 13 changed files with 279 additions and 100 deletions.
diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/common/tasks/RedHat.yml b/core/src/epicli/data/common/ansible/playbooks/roles/common/tasks/RedHat.yml
@@ -6,8 +6,7 @@
 
 - name: Install RedHat family packages
   yum:
-    name: 
-      - epel-release
+    name:
       - libselinux-python
       - libsemanage-python
       - firewalld
@@ -18,9 +17,9 @@
       - nc
       - curl
       - tmux
-      # - fping
-      # - iftop
-      # - htop
+      - fping
+      - iftop
+      - htop
       - vim
       - sysstat
       - python-setuptools
@@ -30,7 +29,7 @@
       - ebtables
       - ethtool
       - telnet
-    update_cache: yes 
+    update_cache: yes
     state: present
   register: result
   retries: 3

diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/docker/tasks/main.yml b/core/src/epicli/data/common/ansible/playbooks/roles/docker/tasks/main.yml
@@ -19,7 +19,7 @@
     state: present
   loop:
       - container-selinux
-      - docker-ce
+      - docker-ce-18.09.9
   when: ansible_os_family == "RedHat"
 
 - include_tasks: configure-docker.yml
diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_common/tasks/load-image.yml b/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_common/tasks/load-image.yml
@@ -1,6 +1,6 @@
 ---
 
-- name: check if image is already loaded
+- name: Check if image is already loaded
   shell: "docker images {{ docker_image.name }} --format {{ '{{' }}.ID{{ '}}' }}"
   register: image_check
   ignore_errors: true

diff --git a/.../playbooks/roles/repository/files/download-requirements/centos-7/download-requirements.sh b/.../playbooks/roles/repository/files/download-requirements/centos-7/download-requirements.sh
@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-# VERSION 1.0.3
+# VERSION 1.2.1
 
 set -euo pipefail
 
@@ -69,7 +69,7 @@ download_file() {
 	local dest_path="$dest_dir/$file_name"
 
 	# wget with --timestamping sometimes failes on AWS with ERROR 403: Forbidden
-	# so we remove existing file to overwrite it
+	# so we remove existing file to overwrite it, to be optimized
 	[[ ! -f $dest_path ]] || remove_file "$dest_path"
 
 	echol "Downloading file: $file"
@@ -103,12 +103,20 @@ download_packages() {
 	local packages="$@"
 
 	if [[ -n $packages ]]; then
-		yumdownloader --quiet --destdir "$dest_dir" $packages || exit_with_error "yumdownloader failed for: $packages"
+		# when using --archlist=x86_64 yumdownloader (yum-utils-1.1.31-52) also downloads i686 packages
+		yumdownloader --quiet --archlist=x86_64 --exclude='*i686' --destdir="$dest_dir" $packages ||
+			exit_with_error "yumdownloader failed for: $packages"
 	fi
 }
 
 echol() {
-	echo -e "$1" |& tee --append $LOG_FILE_PATH
+	echo -e "$@"
+	if [[ $LOG_TO_JOURNAL == 'YES' ]]; then
+		echo -e "$@" | systemd-cat --identifier=$SCRIPT_FILE_NAME
+	else # log to $LOG_FILE_PATH
+		local timestamp=$(date +"%b %e %H:%M:%S")
+		echo -e "${timestamp}: $@" >> "$LOG_FILE_PATH"
+	fi
 }
 
 # params: <repo_id>
@@ -128,11 +136,11 @@ exit_with_error() {
 }
 
 # params: <result_var> <package>
-get_package_dependencies() {
+get_package_dependencies_with_arch() {
 	# $1 reserved for result
 	local package="$2"
 
-	local query_output=$(repoquery --requires --resolve --all --queryformat '%{ui_nevra}' --archlist=x86_64,noarch "$package" 2>&1) ||
+	local query_output=$(repoquery --requires --resolve --queryformat '%{name}.%{arch}' --archlist=x86_64,noarch "$package" 2>&1) ||
 		exit_with_error "repoquery failed for dependencies of package: $package with exit code: $?, output was: $query_output"
 
 	if [[ -z $query_output ]]; then
@@ -141,18 +149,19 @@ get_package_dependencies() {
 		exit_with_error "repoquery failed for dependencies of package: $package, output was: $query_output"
 	fi
 
-	eval $1='$query_output'
+	eval $1='($query_output)'
 }
 
+# desc: get full package name with version and architecture
 # params: <result_var> <package>
-get_package_with_version() {
+get_package_with_version_arch() {
 	# $1 reserved for result
 	local package="$2"
 
-	local query_output=$(repoquery --all --queryformat '%{ui_nevra}' --archlist=x86_64,noarch "$package" 2>&1) ||
+	local query_output=$(repoquery --queryformat '%{ui_nevra}' --archlist=x86_64,noarch "$package" 2>&1) ||
 		exit_with_error "repoquery failed for package: $package with exit code: $?, output was: $query_output"
 
-	# yumdownloader doesn't handle error codes properly if repoquery gets empty output
+	# yumdownloader doesn't set error code if repoquery returns empty output
 	[[ -n $query_output ]] || exit_with_error "repoquery failed: package $package not found"
 	if grep --ignore-case --perl-regexp '\b(?<!-)error(?!-)\b' <<< "$query_output"; then
 		exit_with_error "repoquery failed for package: $package, output was: $query_output"
@@ -163,6 +172,21 @@ get_package_with_version() {
 	eval $1='$query_output'
 }
 
+# params: <result_var> <packages_array>
+get_packages_with_version_arch() {
+	local result_var_name="$1"
+	shift
+	local packages=("$@")
+	local packages_with_version_arch=()
+
+	for package in "${packages[@]}"; do
+		get_package_with_version_arch 'QUERY_OUTPUT' "$package"
+		packages_with_version_arch+=("$QUERY_OUTPUT")
+	done
+
+	eval $result_var_name='("${packages_with_version_arch[@]}")'
+}
+
 # params: <result_var> <group_name> <requirements_file_path>
 get_requirements_from_group() {
 	# $1 reserved for result
@@ -178,6 +202,18 @@ get_requirements_from_group() {
 	eval $1='$requirements_from_group'
 }
 
+# params: <result_var> <array>
+get_unique_array() {
+	local result_var_name="$1"
+	shift
+	local array=("$@")
+
+	# filter out duplicates
+	array=($(echo "${array[@]}" | tr ' ' '\n' | sort -u | tr '\n' ' '))
+
+	eval $result_var_name='("${array[@]}")'
+}
+
 # params: <package_name_or_url> [package_name]
 install_package() {
 	local package_name_or_url="$1"
@@ -269,14 +305,32 @@ usage() {
 
 [ $# -gt 0 ] || usage 1 >&2
 
+# --- Parse arguments ---
+
+POSITIONAL_ARGS=()
+LOG_TO_JOURNAL='NO'
+while [[ $# -gt 0 ]]; do
+case $1 in
+	--log-to-journal)
+	LOG_TO_JOURNAL='YES'
+	shift # past argument
+	;;
+	*) # unknown option
+	POSITIONAL_ARGS+=("$1") # save it in an array for later
+	shift
+	;;
+esac
+done
+set -- "${POSITIONAL_ARGS[@]}" # restore positional arguments
+
 # --- Global variables ---
 
 # dirs
 readonly DOWNLOADS_DIR="$1" # root directory for downloads
 readonly FILES_DIR="$DOWNLOADS_DIR/files"
 readonly PACKAGES_DIR="$DOWNLOADS_DIR/packages"
 readonly IMAGES_DIR="$DOWNLOADS_DIR/images"
-readonly OFFLINE_PREREQ_PACKAGES_DIR="$PACKAGES_DIR/offline-prereqs"
+readonly REPO_PREREQ_PACKAGES_DIR="$PACKAGES_DIR/repo-prereqs"
 readonly SCRIPT_DIR="$(dirname $(readlink -f $0))" # want absolute path
 
 # files
@@ -299,11 +353,11 @@ readonly INSTALLED_PACKAGES_FILE_PATH="$SCRIPT_DIR/${SCRIPT_FILE_NAME}-installed
 
 # --- Parse requirements file ---
 
-# Requirements are grouped using sections: [packages-offline-prereqs], [packages], [files], [images]
-get_requirements_from_group 'OFFLINE_PREREQ_PACKAGES' 'packages-offline-prereqs' "$REQUIREMENTS_FILE_PATH"
-get_requirements_from_group 'PACKAGES'                'packages'                 "$REQUIREMENTS_FILE_PATH"
-get_requirements_from_group 'FILES'                   'files'                    "$REQUIREMENTS_FILE_PATH"
-get_requirements_from_group 'IMAGES'                  'images'                   "$REQUIREMENTS_FILE_PATH"
+# Requirements are grouped using sections: [packages-repo-prereqs], [packages], [files], [images]
+get_requirements_from_group 'REPO_PREREQ_PACKAGES' 'packages-repo-prereqs' "$REQUIREMENTS_FILE_PATH"
+get_requirements_from_group 'PACKAGES'             'packages'              "$REQUIREMENTS_FILE_PATH"
+get_requirements_from_group 'FILES'                'files'                 "$REQUIREMENTS_FILE_PATH"
+get_requirements_from_group 'IMAGES'               'images'                "$REQUIREMENTS_FILE_PATH"
 
 # === Packages ===
 
@@ -422,36 +476,71 @@ if ! is_package_installed 'centos-release-scl'; then
 	install_package 'centos-release-scl'
 fi
 
-# fping package is a part of EPEL repo
+# some packages are from EPEL repo
 if ! is_package_installed 'epel-release'; then
 	install_package 'https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm' 'epel-release'
 fi
 
-echol "Executing: yum -y makecache fast" && yum -y makecache fast
+echol "Executing: yum -y makecache" && yum -y makecache
 
 # --- Download packages ---
 
-create_directory "$OFFLINE_PREREQ_PACKAGES_DIR"
-create_directory "$PACKAGES_DIR"
+# 1) packages required to create repository
+
+create_directory "$REPO_PREREQ_PACKAGES_DIR"
 
-for package in $OFFLINE_PREREQ_PACKAGES; do
+# prepare lists
+PREREQ_PACKAGES=()
+DEPENDENCIES_OF_PREREQ_PACKAGES=()
+for package in $REPO_PREREQ_PACKAGES; do
 	echol "Processing package: $package"
-	get_package_with_version 'QUERY_OUTPUT' "$package"
-	download_packages "$OFFLINE_PREREQ_PACKAGES_DIR" $QUERY_OUTPUT
-	# download package dependencies if exist
-	get_package_dependencies 'QUERY_OUTPUT' "$package"
-	download_packages "$OFFLINE_PREREQ_PACKAGES_DIR" $QUERY_OUTPUT
+	get_package_with_version_arch 'QUERY_OUTPUT' "$package"
+	PREREQ_PACKAGES+=("$QUERY_OUTPUT")
+	get_package_dependencies_with_arch 'DEPENDENCIES' "$package"
+	if [[ ${#DEPENDENCIES[@]} -gt 0 ]]; then
+		for dependency in "${DEPENDENCIES[@]}"; do
+			DEPENDENCIES_OF_PREREQ_PACKAGES+=("$dependency")
+		done
+	fi
 done
 
+# download requirements (fixed versions)
+echol "Downloading repository prerequisite packages (${#PREREQ_PACKAGES[@]})..."
+download_packages "$REPO_PREREQ_PACKAGES_DIR" "${PREREQ_PACKAGES[@]}"
+# download dependencies (latest versions)
+get_unique_array 'DEPENDENCIES' "${DEPENDENCIES_OF_PREREQ_PACKAGES[@]}"
+get_packages_with_version_arch 'DEPENDENCIES' "${DEPENDENCIES[@]}"
+echol "Downloading dependencies of repository prerequisite packages (${#DEPENDENCIES[@]})..."
+download_packages "$REPO_PREREQ_PACKAGES_DIR" "${DEPENDENCIES[@]}"
+
+# 2) non-prerequisite packages
+
+create_directory "$PACKAGES_DIR"
+
+# prepare lists
+NON_PREREQ_PACKAGES=()
+DEPENDENCIES_OF_NON_PREREQ_PACKAGES=()
 for package in $PACKAGES; do
 	echol "Processing package: $package"
-	get_package_with_version 'QUERY_OUTPUT' "$package"
-	download_packages "$PACKAGES_DIR" $QUERY_OUTPUT
-	# download package dependencies if exist
-	get_package_dependencies 'QUERY_OUTPUT' "$package"
-	download_packages "$PACKAGES_DIR" $QUERY_OUTPUT
+	get_package_with_version_arch 'QUERY_OUTPUT' "$package"
+	NON_PREREQ_PACKAGES+=("$QUERY_OUTPUT")
+	get_package_dependencies_with_arch 'DEPENDENCIES' "$package"
+	if [[ ${#DEPENDENCIES[@]} -gt 0 ]]; then
+		for dependency in "${DEPENDENCIES[@]}"; do
+			DEPENDENCIES_OF_NON_PREREQ_PACKAGES+=("$dependency")
+		done
+	fi
 done
 
+# download requirements (fixed versions)
+echol "Downloading packages (${#NON_PREREQ_PACKAGES[@]})..."
+download_packages "$PACKAGES_DIR" "${NON_PREREQ_PACKAGES[@]}"
+# download dependencies (latest versions)
+get_unique_array 'DEPENDENCIES' "${DEPENDENCIES_OF_NON_PREREQ_PACKAGES[@]}"
+get_packages_with_version_arch 'DEPENDENCIES' "${DEPENDENCIES[@]}"
+echol "Downloading dependencies of packages (${#DEPENDENCIES[@]})..."
+download_packages "$PACKAGES_DIR" "${DEPENDENCIES[@]}"
+
 # --- Clean up yum repos ---
 
 remove_added_repos "$ADDED_REPOSITORIES_FILE_PATH"

diff --git a/.../ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.txt b/.../ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.txt
@@ -1,6 +1,6 @@
 # Put this file in the same directory as download script
 
-[packages-offline-prereqs]
+[packages-repo-prereqs]
 createrepo
 httpd
 mod_ssl
@@ -21,7 +21,6 @@ docker-ce-cli-18.09.9
 ebtables
 elasticsearch-curator-5.5.4
 elasticsearch-oss-6.4.0
-epel-release
 erlang-21.3.8.7
 ethtool
 filebeat-6.5.4 # actually it's filebeat-oss
@@ -65,6 +64,7 @@ python-kitchen # for yum-utils
 python-lxml # for java-1.8.0-openjdk-headless
 python-psycopg2
 python-setuptools
+python-slip-dbus # for firewalld
 quota # for nfs-utils
 rabbitmq-server-3.7.10
 rh-haproxy18
@@ -118,4 +118,4 @@ kubernetesui/dashboard:v2.0.0-beta1
 kubernetesui/metrics-scraper:v1.0.0
 calico/cni:v3.8.1
 jboss/keycloak:4.8.3.Final
-rabbitmq:3.7.10
+rabbitmq:3.7.10