diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/common/tasks/RedHat.yml b/core/src/epicli/data/common/ansible/playbooks/roles/common/tasks/RedHat.yml index 8b6d0cdea1..11a3c91d1a 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/common/tasks/RedHat.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/common/tasks/RedHat.yml @@ -6,8 +6,7 @@ - name: Install RedHat family packages yum: - name: - - epel-release + name: - libselinux-python - libsemanage-python - firewalld @@ -18,9 +17,9 @@ - nc - curl - tmux - # - fping - # - iftop - # - htop + - fping + - iftop + - htop - vim - sysstat - python-setuptools @@ -30,7 +29,7 @@ - ebtables - ethtool - telnet - update_cache: yes + update_cache: yes state: present register: result retries: 3 diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/docker/tasks/main.yml b/core/src/epicli/data/common/ansible/playbooks/roles/docker/tasks/main.yml index 5d6c7946fd..d706438955 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/docker/tasks/main.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/docker/tasks/main.yml @@ -19,7 +19,7 @@ state: present loop: - container-selinux - - docker-ce + - docker-ce-18.09.9 when: ansible_os_family == "RedHat" - include_tasks: configure-docker.yml diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_common/tasks/load-image.yml b/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_common/tasks/load-image.yml index 90dd061b8b..e6aa593473 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_common/tasks/load-image.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_common/tasks/load-image.yml @@ -1,6 +1,6 @@ --- -- name: check if image is already loaded +- name: Check if image is already loaded shell: "docker images {{ docker_image.name }} --format {{ '{{' }}.ID{{ '}}' }}" register: image_check ignore_errors: true diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/centos-7/download-requirements.sh b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/centos-7/download-requirements.sh index f450db6f90..e0c862b967 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/centos-7/download-requirements.sh +++ b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/centos-7/download-requirements.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# VERSION 1.0.3 +# VERSION 1.2.1 set -euo pipefail @@ -69,7 +69,7 @@ download_file() { local dest_path="$dest_dir/$file_name" # wget with --timestamping sometimes failes on AWS with ERROR 403: Forbidden - # so we remove existing file to overwrite it + # so we remove existing file to overwrite it, to be optimized [[ ! -f $dest_path ]] || remove_file "$dest_path" echol "Downloading file: $file" @@ -103,12 +103,20 @@ download_packages() { local packages="$@" if [[ -n $packages ]]; then - yumdownloader --quiet --destdir "$dest_dir" $packages || exit_with_error "yumdownloader failed for: $packages" + # when using --archlist=x86_64 yumdownloader (yum-utils-1.1.31-52) also downloads i686 packages + yumdownloader --quiet --archlist=x86_64 --exclude='*i686' --destdir="$dest_dir" $packages || + exit_with_error "yumdownloader failed for: $packages" fi } echol() { - echo -e "$1" |& tee --append $LOG_FILE_PATH + echo -e "$@" + if [[ $LOG_TO_JOURNAL == 'YES' ]]; then + echo -e "$@" | systemd-cat --identifier=$SCRIPT_FILE_NAME + else # log to $LOG_FILE_PATH + local timestamp=$(date +"%b %e %H:%M:%S") + echo -e "${timestamp}: $@" >> "$LOG_FILE_PATH" + fi } # params: @@ -128,11 +136,11 @@ exit_with_error() { } # params: -get_package_dependencies() { +get_package_dependencies_with_arch() { # $1 reserved for result local package="$2" - local query_output=$(repoquery --requires --resolve --all --queryformat '%{ui_nevra}' --archlist=x86_64,noarch "$package" 2>&1) || + local query_output=$(repoquery --requires --resolve --queryformat '%{name}.%{arch}' --archlist=x86_64,noarch "$package" 2>&1) || exit_with_error "repoquery failed for dependencies of package: $package with exit code: $?, output was: $query_output" if [[ -z $query_output ]]; then @@ -141,18 +149,19 @@ get_package_dependencies() { exit_with_error "repoquery failed for dependencies of package: $package, output was: $query_output" fi - eval $1='$query_output' + eval $1='($query_output)' } +# desc: get full package name with version and architecture # params: -get_package_with_version() { +get_package_with_version_arch() { # $1 reserved for result local package="$2" - local query_output=$(repoquery --all --queryformat '%{ui_nevra}' --archlist=x86_64,noarch "$package" 2>&1) || + local query_output=$(repoquery --queryformat '%{ui_nevra}' --archlist=x86_64,noarch "$package" 2>&1) || exit_with_error "repoquery failed for package: $package with exit code: $?, output was: $query_output" - # yumdownloader doesn't handle error codes properly if repoquery gets empty output + # yumdownloader doesn't set error code if repoquery returns empty output [[ -n $query_output ]] || exit_with_error "repoquery failed: package $package not found" if grep --ignore-case --perl-regexp '\b(? +get_packages_with_version_arch() { + local result_var_name="$1" + shift + local packages=("$@") + local packages_with_version_arch=() + + for package in "${packages[@]}"; do + get_package_with_version_arch 'QUERY_OUTPUT' "$package" + packages_with_version_arch+=("$QUERY_OUTPUT") + done + + eval $result_var_name='("${packages_with_version_arch[@]}")' +} + # params: get_requirements_from_group() { # $1 reserved for result @@ -178,6 +202,18 @@ get_requirements_from_group() { eval $1='$requirements_from_group' } +# params: +get_unique_array() { + local result_var_name="$1" + shift + local array=("$@") + + # filter out duplicates + array=($(echo "${array[@]}" | tr ' ' '\n' | sort -u | tr '\n' ' ')) + + eval $result_var_name='("${array[@]}")' +} + # params: [package_name] install_package() { local package_name_or_url="$1" @@ -269,6 +305,24 @@ usage() { [ $# -gt 0 ] || usage 1 >&2 +# --- Parse arguments --- + +POSITIONAL_ARGS=() +LOG_TO_JOURNAL='NO' +while [[ $# -gt 0 ]]; do +case $1 in + --log-to-journal) + LOG_TO_JOURNAL='YES' + shift # past argument + ;; + *) # unknown option + POSITIONAL_ARGS+=("$1") # save it in an array for later + shift + ;; +esac +done +set -- "${POSITIONAL_ARGS[@]}" # restore positional arguments + # --- Global variables --- # dirs @@ -276,7 +330,7 @@ readonly DOWNLOADS_DIR="$1" # root directory for downloads readonly FILES_DIR="$DOWNLOADS_DIR/files" readonly PACKAGES_DIR="$DOWNLOADS_DIR/packages" readonly IMAGES_DIR="$DOWNLOADS_DIR/images" -readonly OFFLINE_PREREQ_PACKAGES_DIR="$PACKAGES_DIR/offline-prereqs" +readonly REPO_PREREQ_PACKAGES_DIR="$PACKAGES_DIR/repo-prereqs" readonly SCRIPT_DIR="$(dirname $(readlink -f $0))" # want absolute path # files @@ -299,11 +353,11 @@ readonly INSTALLED_PACKAGES_FILE_PATH="$SCRIPT_DIR/${SCRIPT_FILE_NAME}-installed # --- Parse requirements file --- -# Requirements are grouped using sections: [packages-offline-prereqs], [packages], [files], [images] -get_requirements_from_group 'OFFLINE_PREREQ_PACKAGES' 'packages-offline-prereqs' "$REQUIREMENTS_FILE_PATH" -get_requirements_from_group 'PACKAGES' 'packages' "$REQUIREMENTS_FILE_PATH" -get_requirements_from_group 'FILES' 'files' "$REQUIREMENTS_FILE_PATH" -get_requirements_from_group 'IMAGES' 'images' "$REQUIREMENTS_FILE_PATH" +# Requirements are grouped using sections: [packages-repo-prereqs], [packages], [files], [images] +get_requirements_from_group 'REPO_PREREQ_PACKAGES' 'packages-repo-prereqs' "$REQUIREMENTS_FILE_PATH" +get_requirements_from_group 'PACKAGES' 'packages' "$REQUIREMENTS_FILE_PATH" +get_requirements_from_group 'FILES' 'files' "$REQUIREMENTS_FILE_PATH" +get_requirements_from_group 'IMAGES' 'images' "$REQUIREMENTS_FILE_PATH" # === Packages === @@ -422,36 +476,71 @@ if ! is_package_installed 'centos-release-scl'; then install_package 'centos-release-scl' fi -# fping package is a part of EPEL repo +# some packages are from EPEL repo if ! is_package_installed 'epel-release'; then install_package 'https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm' 'epel-release' fi -echol "Executing: yum -y makecache fast" && yum -y makecache fast +echol "Executing: yum -y makecache" && yum -y makecache # --- Download packages --- -create_directory "$OFFLINE_PREREQ_PACKAGES_DIR" -create_directory "$PACKAGES_DIR" +# 1) packages required to create repository + +create_directory "$REPO_PREREQ_PACKAGES_DIR" -for package in $OFFLINE_PREREQ_PACKAGES; do +# prepare lists +PREREQ_PACKAGES=() +DEPENDENCIES_OF_PREREQ_PACKAGES=() +for package in $REPO_PREREQ_PACKAGES; do echol "Processing package: $package" - get_package_with_version 'QUERY_OUTPUT' "$package" - download_packages "$OFFLINE_PREREQ_PACKAGES_DIR" $QUERY_OUTPUT - # download package dependencies if exist - get_package_dependencies 'QUERY_OUTPUT' "$package" - download_packages "$OFFLINE_PREREQ_PACKAGES_DIR" $QUERY_OUTPUT + get_package_with_version_arch 'QUERY_OUTPUT' "$package" + PREREQ_PACKAGES+=("$QUERY_OUTPUT") + get_package_dependencies_with_arch 'DEPENDENCIES' "$package" + if [[ ${#DEPENDENCIES[@]} -gt 0 ]]; then + for dependency in "${DEPENDENCIES[@]}"; do + DEPENDENCIES_OF_PREREQ_PACKAGES+=("$dependency") + done + fi done +# download requirements (fixed versions) +echol "Downloading repository prerequisite packages (${#PREREQ_PACKAGES[@]})..." +download_packages "$REPO_PREREQ_PACKAGES_DIR" "${PREREQ_PACKAGES[@]}" +# download dependencies (latest versions) +get_unique_array 'DEPENDENCIES' "${DEPENDENCIES_OF_PREREQ_PACKAGES[@]}" +get_packages_with_version_arch 'DEPENDENCIES' "${DEPENDENCIES[@]}" +echol "Downloading dependencies of repository prerequisite packages (${#DEPENDENCIES[@]})..." +download_packages "$REPO_PREREQ_PACKAGES_DIR" "${DEPENDENCIES[@]}" + +# 2) non-prerequisite packages + +create_directory "$PACKAGES_DIR" + +# prepare lists +NON_PREREQ_PACKAGES=() +DEPENDENCIES_OF_NON_PREREQ_PACKAGES=() for package in $PACKAGES; do echol "Processing package: $package" - get_package_with_version 'QUERY_OUTPUT' "$package" - download_packages "$PACKAGES_DIR" $QUERY_OUTPUT - # download package dependencies if exist - get_package_dependencies 'QUERY_OUTPUT' "$package" - download_packages "$PACKAGES_DIR" $QUERY_OUTPUT + get_package_with_version_arch 'QUERY_OUTPUT' "$package" + NON_PREREQ_PACKAGES+=("$QUERY_OUTPUT") + get_package_dependencies_with_arch 'DEPENDENCIES' "$package" + if [[ ${#DEPENDENCIES[@]} -gt 0 ]]; then + for dependency in "${DEPENDENCIES[@]}"; do + DEPENDENCIES_OF_NON_PREREQ_PACKAGES+=("$dependency") + done + fi done +# download requirements (fixed versions) +echol "Downloading packages (${#NON_PREREQ_PACKAGES[@]})..." +download_packages "$PACKAGES_DIR" "${NON_PREREQ_PACKAGES[@]}" +# download dependencies (latest versions) +get_unique_array 'DEPENDENCIES' "${DEPENDENCIES_OF_NON_PREREQ_PACKAGES[@]}" +get_packages_with_version_arch 'DEPENDENCIES' "${DEPENDENCIES[@]}" +echol "Downloading dependencies of packages (${#DEPENDENCIES[@]})..." +download_packages "$PACKAGES_DIR" "${DEPENDENCIES[@]}" + # --- Clean up yum repos --- remove_added_repos "$ADDED_REPOSITORIES_FILE_PATH" diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.txt b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.txt index 7998ed7a88..db67fc1acf 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.txt +++ b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.txt @@ -1,6 +1,6 @@ # Put this file in the same directory as download script -[packages-offline-prereqs] +[packages-repo-prereqs] createrepo httpd mod_ssl @@ -21,7 +21,6 @@ docker-ce-cli-18.09.9 ebtables elasticsearch-curator-5.5.4 elasticsearch-oss-6.4.0 -epel-release erlang-21.3.8.7 ethtool filebeat-6.5.4 # actually it's filebeat-oss @@ -65,6 +64,7 @@ python-kitchen # for yum-utils python-lxml # for java-1.8.0-openjdk-headless python-psycopg2 python-setuptools +python-slip-dbus # for firewalld quota # for nfs-utils rabbitmq-server-3.7.10 rh-haproxy18 @@ -118,4 +118,4 @@ kubernetesui/dashboard:v2.0.0-beta1 kubernetesui/metrics-scraper:v1.0.0 calico/cni:v3.8.1 jboss/keycloak:4.8.3.Final -rabbitmq:3.7.10 +rabbitmq:3.7.10 \ No newline at end of file diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/download-requirements.sh b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/download-requirements.sh index 4c90429887..20d77982f4 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/download-requirements.sh +++ b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/download-requirements.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# VERSION 1.0.3 +# VERSION 1.2.1 set -euo pipefail @@ -69,7 +69,7 @@ download_file() { local dest_path="$dest_dir/$file_name" # wget with --timestamping sometimes failes on AWS with ERROR 403: Forbidden - # so we remove existing file to overwrite it + # so we remove existing file to overwrite it, to be optimized [[ ! -f $dest_path ]] || remove_file "$dest_path" echol "Downloading file: $file" @@ -103,12 +103,20 @@ download_packages() { local packages="$@" if [[ -n $packages ]]; then - yumdownloader --quiet --destdir "$dest_dir" $packages || exit_with_error "yumdownloader failed for: $packages" + # when using --archlist=x86_64 yumdownloader (yum-utils-1.1.31-52) also downloads i686 packages + yumdownloader --quiet --archlist=x86_64 --exclude='*i686' --destdir="$dest_dir" $packages || + exit_with_error "yumdownloader failed for: $packages" fi } echol() { - echo -e "$1" |& tee --append $LOG_FILE_PATH + echo -e "$@" + if [[ $LOG_TO_JOURNAL == 'YES' ]]; then + echo -e "$@" | systemd-cat --identifier=$SCRIPT_FILE_NAME + else # log to $LOG_FILE_PATH + local timestamp=$(date +"%b %e %H:%M:%S") + echo -e "${timestamp}: $@" >> "$LOG_FILE_PATH" + fi } # params: @@ -127,8 +135,8 @@ exit_with_error() { exit 1 } -# params: # desc: find repo id (set $1) based on given pattern +# params: find_rhel_repo_id() { # $1 reserved for result local rhel_on_prem_repo_id="$2" @@ -145,11 +153,11 @@ find_rhel_repo_id() { } # params: -get_package_dependencies() { +get_package_dependencies_with_arch() { # $1 reserved for result local package="$2" - local query_output=$(repoquery --requires --resolve --all --queryformat '%{ui_nevra}' --archlist=x86_64,noarch "$package" 2>&1) || + local query_output=$(repoquery --requires --resolve --queryformat '%{name}.%{arch}' --archlist=x86_64,noarch "$package" 2>&1) || exit_with_error "repoquery failed for dependencies of package: $package with exit code: $?, output was: $query_output" if [[ -z $query_output ]]; then @@ -158,18 +166,19 @@ get_package_dependencies() { exit_with_error "repoquery failed for dependencies of package: $package, output was: $query_output" fi - eval $1='$query_output' + eval $1='($query_output)' } +# desc: get full package name with version and architecture # params: -get_package_with_version() { +get_package_with_version_arch() { # $1 reserved for result local package="$2" - local query_output=$(repoquery --all --queryformat '%{ui_nevra}' --archlist=x86_64,noarch "$package" 2>&1) || + local query_output=$(repoquery --queryformat '%{ui_nevra}' --archlist=x86_64,noarch "$package" 2>&1) || exit_with_error "repoquery failed for package: $package with exit code: $?, output was: $query_output" - # yumdownloader doesn't handle error codes properly if repoquery gets empty output + # yumdownloader doesn't set error code if repoquery returns empty output [[ -n $query_output ]] || exit_with_error "repoquery failed: package $package not found" if grep --ignore-case --perl-regexp '\b(? +get_packages_with_version_arch() { + local result_var_name="$1" + shift + local packages=("$@") + local packages_with_version_arch=() + + for package in "${packages[@]}"; do + get_package_with_version_arch 'QUERY_OUTPUT' "$package" + packages_with_version_arch+=("$QUERY_OUTPUT") + done + + eval $result_var_name='("${packages_with_version_arch[@]}")' +} + # params: get_requirements_from_group() { # $1 reserved for result @@ -195,6 +219,18 @@ get_requirements_from_group() { eval $1='$requirements_from_group' } +# params: +get_unique_array() { + local result_var_name="$1" + shift + local array=("$@") + + # filter out duplicates + array=($(echo "${array[@]}" | tr ' ' '\n' | sort -u | tr '\n' ' ')) + + eval $result_var_name='("${array[@]}")' +} + # params: [package_name] install_package() { local package_name_or_url="$1" @@ -286,6 +322,24 @@ usage() { [ $# -gt 0 ] || usage 1 >&2 +# --- Parse arguments --- + +POSITIONAL_ARGS=() +LOG_TO_JOURNAL='NO' +while [[ $# -gt 0 ]]; do +case $1 in + --log-to-journal) + LOG_TO_JOURNAL='YES' + shift # past argument + ;; + *) # unknown option + POSITIONAL_ARGS+=("$1") # save it in an array for later + shift + ;; +esac +done +set -- "${POSITIONAL_ARGS[@]}" # restore positional arguments + # --- Global variables --- # dirs @@ -293,7 +347,7 @@ readonly DOWNLOADS_DIR="$1" # root directory for downloads readonly FILES_DIR="$DOWNLOADS_DIR/files" readonly PACKAGES_DIR="$DOWNLOADS_DIR/packages" readonly IMAGES_DIR="$DOWNLOADS_DIR/images" -readonly OFFLINE_PREREQ_PACKAGES_DIR="$PACKAGES_DIR/offline-prereqs" +readonly REPO_PREREQ_PACKAGES_DIR="$PACKAGES_DIR/repo-prereqs" readonly SCRIPT_DIR="$(dirname $(readlink -f $0))" # want absolute path # files @@ -316,11 +370,11 @@ readonly INSTALLED_PACKAGES_FILE_PATH="$SCRIPT_DIR/${SCRIPT_FILE_NAME}-installed # --- Parse requirements file --- -# Requirements are grouped using sections: [packages-offline-prereqs], [packages], [files], [images] -get_requirements_from_group 'OFFLINE_PREREQ_PACKAGES' 'packages-offline-prereqs' "$REQUIREMENTS_FILE_PATH" -get_requirements_from_group 'PACKAGES' 'packages' "$REQUIREMENTS_FILE_PATH" -get_requirements_from_group 'FILES' 'files' "$REQUIREMENTS_FILE_PATH" -get_requirements_from_group 'IMAGES' 'images' "$REQUIREMENTS_FILE_PATH" +# Requirements are grouped using sections: [packages-repo-prereqs], [packages], [files], [images] +get_requirements_from_group 'REPO_PREREQ_PACKAGES' 'packages-repo-prereqs' "$REQUIREMENTS_FILE_PATH" +get_requirements_from_group 'PACKAGES' 'packages' "$REQUIREMENTS_FILE_PATH" +get_requirements_from_group 'FILES' 'files' "$REQUIREMENTS_FILE_PATH" +get_requirements_from_group 'IMAGES' 'images' "$REQUIREMENTS_FILE_PATH" # === Packages === @@ -445,36 +499,71 @@ add_repo_as_file 'kubernetes' "$KUBERNETES_REPO_CONF" add_repo_as_file 'rabbitmq_erlang' "$RABBITMQ_ERLANG_REPO_CONF" add_repo_as_file 'rabbitmq_rabbitmq-server' "$RABBITMQ_SERVER_REPO_CONF" -# fping package is a part of EPEL repo +# some packages are from EPEL repo if ! is_package_installed 'epel-release'; then install_package 'https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm' 'epel-release' fi -echol "Executing: yum -y makecache fast" && yum -y makecache fast +echol "Executing: yum -y makecache" && yum -y makecache # --- Download packages --- -create_directory "$OFFLINE_PREREQ_PACKAGES_DIR" -create_directory "$PACKAGES_DIR" +# 1) packages required to create repository -for package in $OFFLINE_PREREQ_PACKAGES; do +create_directory "$REPO_PREREQ_PACKAGES_DIR" + +# prepare lists +PREREQ_PACKAGES=() +DEPENDENCIES_OF_PREREQ_PACKAGES=() +for package in $REPO_PREREQ_PACKAGES; do echol "Processing package: $package" - get_package_with_version 'QUERY_OUTPUT' "$package" - download_packages "$OFFLINE_PREREQ_PACKAGES_DIR" $QUERY_OUTPUT - # download package dependencies if exist - get_package_dependencies 'QUERY_OUTPUT' "$package" - download_packages "$OFFLINE_PREREQ_PACKAGES_DIR" $QUERY_OUTPUT + get_package_with_version_arch 'QUERY_OUTPUT' "$package" + PREREQ_PACKAGES+=("$QUERY_OUTPUT") + get_package_dependencies_with_arch 'DEPENDENCIES' "$package" + if [[ ${#DEPENDENCIES[@]} -gt 0 ]]; then + for dependency in "${DEPENDENCIES[@]}"; do + DEPENDENCIES_OF_PREREQ_PACKAGES+=("$dependency") + done + fi done +# download requirements (fixed versions) +echol "Downloading repository prerequisite packages (${#PREREQ_PACKAGES[@]})..." +download_packages "$REPO_PREREQ_PACKAGES_DIR" "${PREREQ_PACKAGES[@]}" +# download dependencies (latest versions) +get_unique_array 'DEPENDENCIES' "${DEPENDENCIES_OF_PREREQ_PACKAGES[@]}" +get_packages_with_version_arch 'DEPENDENCIES' "${DEPENDENCIES[@]}" +echol "Downloading dependencies of repository prerequisite packages (${#DEPENDENCIES[@]})..." +download_packages "$REPO_PREREQ_PACKAGES_DIR" "${DEPENDENCIES[@]}" + +# 2) non-prerequisite packages + +create_directory "$PACKAGES_DIR" + +# prepare lists +NON_PREREQ_PACKAGES=() +DEPENDENCIES_OF_NON_PREREQ_PACKAGES=() for package in $PACKAGES; do echol "Processing package: $package" - get_package_with_version 'QUERY_OUTPUT' "$package" - download_packages "$PACKAGES_DIR" $QUERY_OUTPUT - # download package dependencies if exist - get_package_dependencies 'QUERY_OUTPUT' "$package" - download_packages "$PACKAGES_DIR" $QUERY_OUTPUT + get_package_with_version_arch 'QUERY_OUTPUT' "$package" + NON_PREREQ_PACKAGES+=("$QUERY_OUTPUT") + get_package_dependencies_with_arch 'DEPENDENCIES' "$package" + if [[ ${#DEPENDENCIES[@]} -gt 0 ]]; then + for dependency in "${DEPENDENCIES[@]}"; do + DEPENDENCIES_OF_NON_PREREQ_PACKAGES+=("$dependency") + done + fi done +# download requirements (fixed versions) +echol "Downloading packages (${#NON_PREREQ_PACKAGES[@]})..." +download_packages "$PACKAGES_DIR" "${NON_PREREQ_PACKAGES[@]}" +# download dependencies (latest versions) +get_unique_array 'DEPENDENCIES' "${DEPENDENCIES_OF_NON_PREREQ_PACKAGES[@]}" +get_packages_with_version_arch 'DEPENDENCIES' "${DEPENDENCIES[@]}" +echol "Downloading dependencies of packages (${#DEPENDENCIES[@]})..." +download_packages "$PACKAGES_DIR" "${DEPENDENCIES[@]}" + # --- Clean up yum repos --- remove_added_repos "$ADDED_REPOSITORIES_FILE_PATH" diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/requirements.txt b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/requirements.txt index 7998ed7a88..db67fc1acf 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/requirements.txt +++ b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/requirements.txt @@ -1,6 +1,6 @@ # Put this file in the same directory as download script -[packages-offline-prereqs] +[packages-repo-prereqs] createrepo httpd mod_ssl @@ -21,7 +21,6 @@ docker-ce-cli-18.09.9 ebtables elasticsearch-curator-5.5.4 elasticsearch-oss-6.4.0 -epel-release erlang-21.3.8.7 ethtool filebeat-6.5.4 # actually it's filebeat-oss @@ -65,6 +64,7 @@ python-kitchen # for yum-utils python-lxml # for java-1.8.0-openjdk-headless python-psycopg2 python-setuptools +python-slip-dbus # for firewalld quota # for nfs-utils rabbitmq-server-3.7.10 rh-haproxy18 @@ -118,4 +118,4 @@ kubernetesui/dashboard:v2.0.0-beta1 kubernetesui/metrics-scraper:v1.0.0 calico/cni:v3.8.1 jboss/keycloak:4.8.3.Final -rabbitmq:3.7.10 +rabbitmq:3.7.10 \ No newline at end of file diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/server/RedHat/create-repository.sh b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/server/RedHat/create-repository.sh index 243841ceb0..336fa6c4f2 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/server/RedHat/create-repository.sh +++ b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/server/RedHat/create-repository.sh @@ -4,7 +4,7 @@ EPI_REPO_SERVER_PATH=$1 # /var/www/html/epirepo is the default IS_OFFLINE_MODE=$2 if $IS_OFFLINE_MODE = true; then - yum install -y $(ls $EPI_REPO_SERVER_PATH/packages/offline_prereqs/*.rpm) #TODO: to test + yum install -y $(ls $EPI_REPO_SERVER_PATH/packages/repo_prereqs/*.rpm) #TODO: to test else yum install -y httpd createrepo yum-utils fi diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/repository/tasks/download-requirements.yml b/core/src/epicli/data/common/ansible/playbooks/roles/repository/tasks/download-requirements.yml index 0c2bd2c531..a2f8e032c9 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/repository/tasks/download-requirements.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/repository/tasks/download-requirements.yml @@ -20,8 +20,10 @@ path: /tmp/epi-download-requirements/download-requirements-done.flag register: stat_result -- name: Run download-requirements script, it can take a long time - shell: /tmp/epi-download-requirements/download-requirements.sh /var/www/html/epirepo +- name: |- + Run download-requirements script, it can take a long time + You can check progress on repository host with: journalctl -f -t download-requirements.sh + shell: /tmp/epi-download-requirements/download-requirements.sh /var/www/html/epirepo --log-to-journal when: not stat_result.stat.exists - name: Create flag file to not re-download requirements next time diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/zookeeper/tasks/main.yml b/core/src/epicli/data/common/ansible/playbooks/roles/zookeeper/tasks/main.yml index 5bdeddb214..bbbd916fe3 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/zookeeper/tasks/main.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/zookeeper/tasks/main.yml @@ -76,7 +76,7 @@ group: "{{ zookeeper_group }}" check_mode: false -- name: "Create zookeeper {{item}} directory." +- name: Create directories file: path: "{{ item }}" state: directory @@ -88,7 +88,7 @@ - "{{ zookeeper_install_dir }}/conf" - "/etc/zookeeper/conf" -- name: Systemd script. +- name: Create zookeeper service template: src: zookeeper.service.j2 dest: /lib/systemd/system/zookeeper.service diff --git a/core/src/epicli/data/common/defaults/configuration/feature-mapping.yml b/core/src/epicli/data/common/defaults/configuration/feature-mapping.yml index 436bfb070a..1c913df5f2 100644 --- a/core/src/epicli/data/common/defaults/configuration/feature-mapping.yml +++ b/core/src/epicli/data/common/defaults/configuration/feature-mapping.yml @@ -5,7 +5,7 @@ name: default specification: available_roles: - name: repository - enabled: yes + enabled: no # disabled by purpose, there is no ansible/repository.yml - name: kubernetes-master enabled: yes - name: kubernetes-node diff --git a/core/src/epicli/data/common/defaults/configuration/jmx-exporter.yml b/core/src/epicli/data/common/defaults/configuration/jmx-exporter.yml index 06228beb04..eb98933ad8 100644 --- a/core/src/epicli/data/common/defaults/configuration/jmx-exporter.yml +++ b/core/src/epicli/data/common/defaults/configuration/jmx-exporter.yml @@ -3,7 +3,7 @@ version: 0.4.0 title: "JMX exporter" name: default specification: - file_name: "jmx_prometheus_javaagent-0.3.1.jar" + file_name: "jmx_prometheus_javaagent-0.12.0.jar" jmx_path: /opt/jmx-exporter/jmx_prometheus_javaagent.jar # Changing it requires also change for same variable in Kafka and Zookeeper configs. # Todo Zookeeper and Kafka to use this variable jmx_jars_directory: /opt/jmx-exporter/jars jmx_exporter_user: jmx-exporter diff --git a/core/src/epicli/data/common/defaults/configuration/kubernetes-master.yml b/core/src/epicli/data/common/defaults/configuration/kubernetes-master.yml index e06a91db76..c6a7b7b51f 100644 --- a/core/src/epicli/data/common/defaults/configuration/kubernetes-master.yml +++ b/core/src/epicli/data/common/defaults/configuration/kubernetes-master.yml @@ -6,37 +6,37 @@ specification: version: 1.14.6 images_to_load: - name: "k8s.gcr.io/kube-apiserver:v1.14.6" - file_name: kube-apiserver:v1.14.6.tar + file_name: kube-apiserver-v1.14.6.tar - name: "k8s.gcr.io/kube-controller-manager:v1.14.6" - file_name: kube-controller-manager:v1.14.6.tar + file_name: kube-controller-manager-v1.14.6.tar - name: "k8s.gcr.io/kube-scheduler:v1.14.6" - file_name: kube-scheduler:v1.14.6.tar + file_name: kube-scheduler-v1.14.6.tar - name: "k8s.gcr.io/kube-proxy:v1.14.6" - file_name: kube-proxy:v1.14.6.tar + file_name: kube-proxy-v1.14.6.tar - name: "k8s.gcr.io/pause:3.1" - file_name: pause:3.1.tar + file_name: pause-3.1.tar - name: "k8s.gcr.io/etcd:3.3.10" - file_name: etcd:3.3.10.tar + file_name: etcd-3.3.10.tar - name: "k8s.gcr.io/coredns:1.3.1" - file_name: coredns:1.3.1.tar + file_name: coredns-1.3.1.tar - name: "quay.io/coreos/flannel:v0.11.0-amd64" - file_name: flannel:v0.11.0-amd64.tar + file_name: flannel-v0.11.0-amd64.tar - name: "quay.io/coreos/flannel:v0.11.0" - file_name: flannel:v0.11.0.tar + file_name: flannel-v0.11.0.tar - name: "calico/node:v3.8.1" - file_name: node:v3.8.1.tar + file_name: node-v3.8.1.tar - name: "calico/pod2daemon-flexvol:v3.8.1" - file_name: pod2daemon-flexvol:v3.8.1.tar + file_name: pod2daemon-flexvol-v3.8.1.tar - name: "kubernetesui/dashboard:v2.0.0-beta1" - file_name: dashboard:v2.0.0-beta1.tar + file_name: dashboard-v2.0.0-beta1.tar - name: "kubernetesui/metrics-scraper:v1.0.0" - file_name: metrics-scraper:v1.0.0.tar + file_name: metrics-scraper-v1.0.0.tar - name: "calico/cni:v3.8.1" - file_name: cni:v3.8.1.tar + file_name: cni-v3.8.1.tar - name: "jboss/keycloak:4.8.3.Final" - file_name: keycloak:4.8.3.Final.tar + file_name: keycloak-4.8.3.Final.tar - name: "rabbitmq:3.7.10" - file_name: rabbitmq:3.7.10.tar + file_name: rabbitmq-3.7.10.tar allow_pods_on_master: False storage: