From 57dc8e0c50d610cf52001f509b3a08a3b4d5f892 Mon Sep 17 00:00:00 2001 From: to-bar <46519524+to-bar@users.noreply.github.com> Date: Thu, 12 Sep 2019 16:10:11 +0200 Subject: [PATCH] Download all packages + fix for admin_user (#412) --- .../tasks/download_all_files_to_cache.yml | 6 +-- .../tasks/download_all_packages_to_cache.yml | 44 +++++++++++++++++++ .../roles/download/tasks/download_file.yml | 6 ++- .../download/tasks/download_file_worker.yml | 15 +++---- .../roles/download/tasks/download_package.yml | 13 +++--- 5 files changed, 65 insertions(+), 19 deletions(-) create mode 100644 core/src/epicli/data/common/ansible/playbooks/roles/download/tasks/download_all_packages_to_cache.yml diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/download/tasks/download_all_files_to_cache.yml b/core/src/epicli/data/common/ansible/playbooks/roles/download/tasks/download_all_files_to_cache.yml index 1ef4fd7839..3df472fac7 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/download/tasks/download_all_files_to_cache.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/download/tasks/download_all_files_to_cache.yml @@ -8,9 +8,9 @@ set_fact: files: >- {{ files + [ item | combine(specification.downloads.files[item.id][item.arch]) - | combine({ 'cached_file_path': specification.local_downloads_dir - + '/' + relative_dest_path }) - | combine({ 'checksum_algorithm': 'sha256' if (item.sha256 is defined) + | combine({ 'cached_file_path': specification.local_downloads_dir + '/' + + relative_dest_path, + 'checksum_algorithm': 'sha256' if (item.sha256 is defined) else 'sha512' }) ] }} vars: relative_dest_path: >- diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/download/tasks/download_all_packages_to_cache.yml b/core/src/epicli/data/common/ansible/playbooks/roles/download/tasks/download_all_packages_to_cache.yml new file mode 100644 index 0000000000..47c5847092 --- /dev/null +++ b/core/src/epicli/data/common/ansible/playbooks/roles/download/tasks/download_all_packages_to_cache.yml @@ -0,0 +1,44 @@ +--- +- name: Prepare list of packages to download + block: + - name: download_all_packages_to_cache | Clear list + set_fact: + packages: [] + - name: download_all_packages_to_cache | Prepare list of packages + set_fact: + packages: >- + {{ packages + [ item | combine({ 'cached_file_path': specification.local_downloads_dir + '/' + + relative_dest_path, + 'checksum_algorithm': 'sha256' if item.sha256 is defined + else 'sha512', + 'distro_version': item.distro_version | string, + 'version': item.version | string }) ] }} + vars: + relative_dest_path: >- + {{ 'packages/' + item.arch + '/' + item.distro + '/' + item.distro_version + '/' + (item.url | basename) }} + loop: >- + [ {% for package_id in specification.downloads.packages.keys() %} + {% for arch in specification.downloads.packages[package_id].keys() %} + {% for package_item in specification.downloads.packages[package_id][arch] if 'url' in package_item %} + {% for distro in ([ package_item.distros ] if package_item.distros is string + else package_item.distros) %} + { 'arch': '{{ arch }}', 'distro': '{{ distro }}', + 'distro_version': '{{ package_item.distro_version }}', + 'id': '{{ package_id }}', + {% if 'sha256' in package_item %} + 'sha256': '{{ package_item.sha256 }}', + {% else %} + 'sha512': '{{ package_item.sha512 }}', + {% endif %} + 'url': '{{ package_item.url }}', + 'version': '{{ package_item.version }}' }, + {% endfor %} + {% endfor %} + {% endfor %} + {% endfor %} ] + +- name: download_all_packages_to_cache | Download packages + include_tasks: download_file_to_cache.yml + vars: + file_arg: "{{ item }}" + loop: "{{ packages }}" \ No newline at end of file diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/download/tasks/download_file.yml b/core/src/epicli/data/common/ansible/playbooks/roles/download/tasks/download_file.yml index b7b4a5a351..4106d592dc 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/download/tasks/download_file.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/download/tasks/download_file.yml @@ -21,7 +21,8 @@ {{ file_items | selectattr('version', 'match', version_to_select) | first }} vars: version_to_select: >- - {{ '.*' if (file_arg.version is not defined) else '^' + (file_arg.version | string) + '$' }} + {{ '.*' if file_arg.version is not defined + else '^' + (file_arg.version | string) + '$' }} - name: download_file | Set dynamic properties set_fact: @@ -30,7 +31,8 @@ | combine(file_arg) | combine({ 'arch': arch, 'cached_file_path': specification.local_downloads_dir + '/' + relative_dest_path, - 'checksum_algorithm': 'sha256' if (file.sha256 is defined) else 'sha512', + 'checksum_algorithm': 'sha256' if file.sha256 is defined + else 'sha512', 'dest_path': specification.remote_downloads_dir + '/' + relative_dest_path, 'version': file.version | string }) }} vars: diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/download/tasks/download_file_worker.yml b/core/src/epicli/data/common/ansible/playbooks/roles/download/tasks/download_file_worker.yml index 61571dee3e..7d249f5bb8 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/download/tasks/download_file_worker.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/download/tasks/download_file_worker.yml @@ -30,13 +30,10 @@ - not is_file_in_cache - file_arg.url is defined -- name: Get admin user and his group +- name: Set admin user's group block: - - name: download_file_worker | Get admin user - set_fact: - admin_user: "{{ admin_user.name }}" - name: download_file_worker | Get admin user's group - command: "id -g -n {{ admin_user }}" + command: "id -g -n {{ admin_user.name }}" register: result changed_when: false - name: download_file_worker | Set admin user's group @@ -45,13 +42,13 @@ become: false when: - not is_file_already_in_dest - - admin_user is not defined or admin_group is not defined + - admin_group is not defined - name: download_file_worker | Create directory '{{ file_arg.dest_path | dirname }}' file: path: "{{ file_arg.dest_path | dirname }}" state: directory - owner: "{{ admin_user }}" + owner: "{{ admin_user.name }}" group: "{{ admin_group }}" become: true when: @@ -64,7 +61,7 @@ msg: - "Copy from: {{ file_arg.cached_file_path }}" - "To: {{ file_arg.dest_path }}" - - name: download_file_worker | Copy '{{ file_arg.id }}' v{{ file_arg.version }} from cache on Ansible host + - name: download_file_worker | Copy '{{ file_arg.id }}' in version '{{ file_arg.version }}' from cache on Ansible host copy: src: "{{ file_arg.cached_file_path }}" dest: "{{ file_arg.dest_path }}" @@ -85,7 +82,7 @@ msg: - "Download from: {{ file_arg.url }}" - "To: {{ file_arg.dest_path }}" - - name: download_file_worker | Download '{{ file_arg.id }}' v{{ file_arg.version }} from direct source + - name: download_file_worker | Download '{{ file_arg.id }}' in version '{{ file_arg.version }}' from direct source get_url: url: "{{ file_arg.url }}" dest: "{{ file_arg.dest_path }}" diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/download/tasks/download_package.yml b/core/src/epicli/data/common/ansible/playbooks/roles/download/tasks/download_package.yml index a6b0efd0d8..309d235cf5 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/download/tasks/download_package.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/download/tasks/download_package.yml @@ -5,7 +5,7 @@ os_distribution: >- {{ (ansible_distribution == "Oracle Linux") | ternary('RedHat', ansible_distribution) }} os_distribution_version: >- - {{ ansible_distribution_major_version if (ansible_os_family == "RedHat") + {{ ansible_distribution_major_version if ansible_os_family == "RedHat" else ansible_distribution_version }} - name: Normalize properties from configuration @@ -21,7 +21,7 @@ normalized_package_item: >- {{ package_item | combine({ 'distro_version': package_item.distro_version | string }) | combine({ 'version': package_item.version | string }) - | combine({ 'distros': [ package_item.distros ] if (package_item.distros is string) + | combine({ 'distros': [ package_item.distros ] if package_item.distros is string else package_item.distros }) }} loop: "{{ specification.downloads.packages[package_arg.id][arch] }}" loop_control: @@ -36,7 +36,8 @@ | first }} vars: version_to_select: >- - {{ '.*' if (package_arg.version is not defined) else '^' + (package_arg.version | string) + '$' }} + {{ '.*' if package_arg.version is not defined + else '^' + (package_arg.version | string) + '$' }} - name: download_package | Set dynamic properties and filter out unneeded set_fact: @@ -46,7 +47,8 @@ | combine(package_arg) | combine({ 'arch': arch, 'cached_file_path': specification.local_downloads_dir + '/' + relative_dest_path, - 'checksum_algorithm': 'sha256' if (file.sha256 is defined) else 'sha512', + 'checksum_algorithm': 'sha256' if file.sha256 is defined + else 'sha512', 'dest_path': specification.remote_downloads_dir + '/' + relative_dest_path, 'os_distribution': os_distribution, 'os_distribution_version': os_distribution_version, @@ -54,7 +56,8 @@ vars: relative_dest_path: >- {{ 'packages/' + arch + '/' + os_distribution + '/' + os_distribution_version + '/' - + (file.url | basename) if file.url is defined else file.file_name }} + + (file.url | basename) if file.url is defined + else file.file_name }} - name: download_package | Download '{{ file.id }}' v{{ file.version }} include_tasks: download_file_worker.yml