Download latest versions of dependencies hitachienergy#536

to-bar · Oct 4, 2019 · 617bb8d · 617bb8d
1 parent 8fdf2ed
commit 617bb8d
Show file tree

Hide file tree

Showing 4 changed files with 98 additions and 36 deletions.
diff --git a/.../playbooks/roles/repository/files/download-requirements/centos-7/download-requirements.sh b/.../playbooks/roles/repository/files/download-requirements/centos-7/download-requirements.sh
@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-# VERSION 1.1.0
+# VERSION 1.2.0
 
 set -euo pipefail
 
@@ -134,11 +134,11 @@ exit_with_error() {
 }
 
 # params: <result_var> <package>
-get_package_dependencies() {
+get_package_dependencies_names() {
 	# $1 reserved for result
 	local package="$2"
 
-	local query_output=$(repoquery --requires --resolve --all --queryformat '%{ui_nevra}' --archlist=x86_64,noarch "$package" 2>&1) ||
+	local query_output=$(repoquery --requires --resolve --queryformat '%{name}' --archlist=x86_64,noarch "$package" 2>&1) ||
 		exit_with_error "repoquery failed for dependencies of package: $package with exit code: $?, output was: $query_output"
 
 	if [[ -z $query_output ]]; then
@@ -155,7 +155,7 @@ get_package_with_version() {
 	# $1 reserved for result
 	local package="$2"
 
-	local query_output=$(repoquery --all --queryformat '%{ui_nevra}' --archlist=x86_64,noarch "$package" 2>&1) ||
+	local query_output=$(repoquery --queryformat '%{ui_nevra}' --archlist=x86_64,noarch "$package" 2>&1) ||
 		exit_with_error "repoquery failed for package: $package with exit code: $?, output was: $query_output"
 
 	# yumdownloader doesn't handle error codes properly if repoquery gets empty output
@@ -169,6 +169,24 @@ get_package_with_version() {
 	eval $1='$query_output'
 }
 
+# params: <result_var> <array_with_dependencies_names>
+get_packages_dependencies_with_versions() {
+	result_var_name="$1"
+	shift
+	local dependencies_names_arr=("$@")
+	local dependencies_with_versions=()
+
+	# filter out duplicates
+	dependencies_names_arr=($(echo "${dependencies_names_arr[@]}" | tr ' ' '\n' | sort -u | tr '\n' ' '))
+
+	for package in "${dependencies_names_arr[@]}"; do
+		get_package_with_version 'QUERY_OUTPUT' "$package" # full package name with version and architecture
+		dependencies_with_versions+=("$QUERY_OUTPUT")
+	done
+
+	eval $result_var_name='"${dependencies_with_versions[@]}"'
+}
+
 # params: <result_var> <group_name> <requirements_file_path>
 get_requirements_from_group() {
 	# $1 reserved for result
@@ -455,27 +473,46 @@ echol "Executing: yum -y makecache" && yum -y makecache
 
 # --- Download packages ---
 
+# 1) packages required to create repository
+
 create_directory "$OFFLINE_PREREQ_PACKAGES_DIR"
-create_directory "$PACKAGES_DIR"
 
+DEPENDENCIES_OF_ALL_PREREQ_PACKAGES=()
 for package in $OFFLINE_PREREQ_PACKAGES; do
 	echol "Processing package: $package"
 	get_package_with_version 'QUERY_OUTPUT' "$package"
 	download_packages "$OFFLINE_PREREQ_PACKAGES_DIR" $QUERY_OUTPUT
-	# download package dependencies if exist
-	get_package_dependencies 'QUERY_OUTPUT' "$package"
-	download_packages "$OFFLINE_PREREQ_PACKAGES_DIR" $QUERY_OUTPUT
+	get_package_dependencies_names 'QUERY_OUTPUT' "$package"
+	for depenency_name in $QUERY_OUTPUT; do
+		DEPENDENCIES_OF_ALL_PREREQ_PACKAGES+=("$depenency_name")
+	done
 done
 
+# download dependencies (latest versions)
+get_packages_dependencies_with_versions 'DEPENDENCIES' "${DEPENDENCIES_OF_ALL_PREREQ_PACKAGES[@]}"
+echol "Downloading dependencies of prerequisite packages (${#DEPENDENCIES_OF_ALL_PREREQ_PACKAGES[@]})..."
+download_packages "$OFFLINE_PREREQ_PACKAGES_DIR" $DEPENDENCIES
+
+# 2) non-prerequisite packages
+
+create_directory "$PACKAGES_DIR"
+
+DEPENDENCIES_OF_ALL_NON_PREREQ_PACKAGES=()
 for package in $PACKAGES; do
 	echol "Processing package: $package"
 	get_package_with_version 'QUERY_OUTPUT' "$package"
 	download_packages "$PACKAGES_DIR" $QUERY_OUTPUT
-	# download package dependencies if exist
-	get_package_dependencies 'QUERY_OUTPUT' "$package"
-	download_packages "$PACKAGES_DIR" $QUERY_OUTPUT
+	get_package_dependencies_names 'QUERY_OUTPUT' "$package"
+	for depenency_name in $QUERY_OUTPUT; do
+		DEPENDENCIES_OF_ALL_NON_PREREQ_PACKAGES+=("$depenency_name")
+	done
 done
 
+# download dependencies (latest versions)
+get_packages_dependencies_with_versions 'DEPENDENCIES' "${DEPENDENCIES_OF_ALL_NON_PREREQ_PACKAGES[@]}"
+echol "Downloading dependencies of all non-prerequisite packages (${#DEPENDENCIES_OF_ALL_NON_PREREQ_PACKAGES[@]})..."
+download_packages "$PACKAGES_DIR" $DEPENDENCIES
+
 # --- Clean up yum repos ---
 
 remove_added_repos "$ADDED_REPOSITORIES_FILE_PATH"

diff --git a/.../ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.txt b/.../ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.txt
@@ -21,7 +21,6 @@ docker-ce-cli-18.09.9
 ebtables
 elasticsearch-curator-5.5.4
 elasticsearch-oss-6.4.0
-epel-release
 erlang-21.3.8.7
 ethtool
 filebeat-6.5.4 # actually it's filebeat-oss
@@ -42,12 +41,8 @@ kubectl-1.14.6
 kubelet-1.14.6
 kubernetes-cni-0.7.5
 libini_config # for nfs-utils
-libldb # for samba-client
-libtdb # for samba-client
 libselinux-python
 libsemanage-python
-libsmbclient # for samba-client
-libtevent # for cifs-utils
 libX11 # for grafana
 libxcb # for grafana
 libXcursor # for grafana
@@ -84,7 +79,6 @@ rh-postgresql10-postgresql-syspaths
 samba-client
 samba-client-libs # for samba-client
 samba-common
-samba-common-libs # for cifs-utils
 socat
 sysstat
 tar
@@ -124,4 +118,4 @@ kubernetesui/dashboard:v2.0.0-beta1
 kubernetesui/metrics-scraper:v1.0.0
 calico/cni:v3.8.1
 jboss/keycloak:4.8.3.Final
-rabbitmq:3.7.10
+rabbitmq:3.7.10
diff --git a/.../playbooks/roles/repository/files/download-requirements/redhat-7/download-requirements.sh b/.../playbooks/roles/repository/files/download-requirements/redhat-7/download-requirements.sh
@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-# VERSION 1.1.0
+# VERSION 1.2.0
 
 set -euo pipefail
 
@@ -151,11 +151,11 @@ find_rhel_repo_id() {
 }
 
 # params: <result_var> <package>
-get_package_dependencies() {
+get_package_dependencies_names() {
 	# $1 reserved for result
 	local package="$2"
 
-	local query_output=$(repoquery --requires --resolve --all --queryformat '%{ui_nevra}' --archlist=x86_64,noarch "$package" 2>&1) ||
+	local query_output=$(repoquery --requires --resolve --queryformat '%{name}' --archlist=x86_64,noarch "$package" 2>&1) ||
 		exit_with_error "repoquery failed for dependencies of package: $package with exit code: $?, output was: $query_output"
 
 	if [[ -z $query_output ]]; then
@@ -172,7 +172,7 @@ get_package_with_version() {
 	# $1 reserved for result
 	local package="$2"
 
-	local query_output=$(repoquery --all --queryformat '%{ui_nevra}' --archlist=x86_64,noarch "$package" 2>&1) ||
+	local query_output=$(repoquery --queryformat '%{ui_nevra}' --archlist=x86_64,noarch "$package" 2>&1) ||
 		exit_with_error "repoquery failed for package: $package with exit code: $?, output was: $query_output"
 
 	# yumdownloader doesn't handle error codes properly if repoquery gets empty output
@@ -186,6 +186,24 @@ get_package_with_version() {
 	eval $1='$query_output'
 }
 
+# params: <result_var> <array_with_dependencies_names>
+get_packages_dependencies_with_versions() {
+	result_var_name="$1"
+	shift
+	local dependencies_names_arr=("$@")
+	local dependencies_with_versions=()
+
+	# filter out duplicates
+	dependencies_names_arr=($(echo "${dependencies_names_arr[@]}" | tr ' ' '\n' | sort -u | tr '\n' ' '))
+
+	for package in "${dependencies_names_arr[@]}"; do
+		get_package_with_version 'QUERY_OUTPUT' "$package" # full package name with version and architecture
+		dependencies_with_versions+=("$QUERY_OUTPUT")
+	done
+
+	eval $result_var_name='"${dependencies_with_versions[@]}"'
+}
+
 # params: <result_var> <group_name> <requirements_file_path>
 get_requirements_from_group() {
 	# $1 reserved for result
@@ -478,27 +496,46 @@ echol "Executing: yum -y makecache" && yum -y makecache
 
 # --- Download packages ---
 
+# 1) packages required to create repository
+
 create_directory "$OFFLINE_PREREQ_PACKAGES_DIR"
-create_directory "$PACKAGES_DIR"
 
+DEPENDENCIES_OF_ALL_PREREQ_PACKAGES=()
 for package in $OFFLINE_PREREQ_PACKAGES; do
 	echol "Processing package: $package"
 	get_package_with_version 'QUERY_OUTPUT' "$package"
 	download_packages "$OFFLINE_PREREQ_PACKAGES_DIR" $QUERY_OUTPUT
-	# download package dependencies if exist
-	get_package_dependencies 'QUERY_OUTPUT' "$package"
-	download_packages "$OFFLINE_PREREQ_PACKAGES_DIR" $QUERY_OUTPUT
+	get_package_dependencies_names 'QUERY_OUTPUT' "$package"
+	for depenency_name in $QUERY_OUTPUT; do
+		DEPENDENCIES_OF_ALL_PREREQ_PACKAGES+=("$depenency_name")
+	done
 done
 
+# download dependencies (latest versions)
+get_packages_dependencies_with_versions 'DEPENDENCIES' "${DEPENDENCIES_OF_ALL_PREREQ_PACKAGES[@]}"
+echol "Downloading dependencies of prerequisite packages (${#DEPENDENCIES_OF_ALL_PREREQ_PACKAGES[@]})..."
+download_packages "$OFFLINE_PREREQ_PACKAGES_DIR" $DEPENDENCIES
+
+# 2) non-prerequisite packages
+
+create_directory "$PACKAGES_DIR"
+
+DEPENDENCIES_OF_ALL_NON_PREREQ_PACKAGES=()
 for package in $PACKAGES; do
 	echol "Processing package: $package"
 	get_package_with_version 'QUERY_OUTPUT' "$package"
 	download_packages "$PACKAGES_DIR" $QUERY_OUTPUT
-	# download package dependencies if exist
-	get_package_dependencies 'QUERY_OUTPUT' "$package"
-	download_packages "$PACKAGES_DIR" $QUERY_OUTPUT
+	get_package_dependencies_names 'QUERY_OUTPUT' "$package"
+	for depenency_name in $QUERY_OUTPUT; do
+		DEPENDENCIES_OF_ALL_NON_PREREQ_PACKAGES+=("$depenency_name")
+	done
 done
 
+# download dependencies (latest versions)
+get_packages_dependencies_with_versions 'DEPENDENCIES' "${DEPENDENCIES_OF_ALL_NON_PREREQ_PACKAGES[@]}"
+echol "Downloading dependencies of all non-prerequisite packages (${#DEPENDENCIES_OF_ALL_NON_PREREQ_PACKAGES[@]})..."
+download_packages "$PACKAGES_DIR" $DEPENDENCIES
+
 # --- Clean up yum repos ---
 
 remove_added_repos "$ADDED_REPOSITORIES_FILE_PATH"

diff --git a/.../ansible/playbooks/roles/repository/files/download-requirements/redhat-7/requirements.txt b/.../ansible/playbooks/roles/repository/files/download-requirements/redhat-7/requirements.txt
@@ -21,7 +21,6 @@ docker-ce-cli-18.09.9
 ebtables
 elasticsearch-curator-5.5.4
 elasticsearch-oss-6.4.0
-epel-release
 erlang-21.3.8.7
 ethtool
 filebeat-6.5.4 # actually it's filebeat-oss
@@ -42,12 +41,8 @@ kubectl-1.14.6
 kubelet-1.14.6
 kubernetes-cni-0.7.5
 libini_config # for nfs-utils
-libldb # for samba-client
-libtdb # for samba-client
 libselinux-python
 libsemanage-python
-libsmbclient # for samba-client
-libtevent # for cifs-utils
 libX11 # for grafana
 libxcb # for grafana
 libXcursor # for grafana
@@ -84,7 +79,6 @@ rh-postgresql10-postgresql-syspaths
 samba-client
 samba-client-libs # for samba-client
 samba-common
-samba-common-libs # for cifs-utils
 socat
 sysstat
 tar
@@ -124,4 +118,4 @@ kubernetesui/dashboard:v2.0.0-beta1
 kubernetesui/metrics-scraper:v1.0.0
 calico/cni:v3.8.1
 jboss/keycloak:4.8.3.Final
-rabbitmq:3.7.10
+rabbitmq:3.7.10