From ccb306dabaec514f31a478c67c048da43c5a908b Mon Sep 17 00:00:00 2001 From: N Date: Tue, 18 Sep 2018 10:27:07 +0100 Subject: [PATCH] Improve packages.archive feature. Include in pillar.example/CI --- .kitchen.yml | 3 +- README.rst | 4 +- packages/archives.sls | 91 ++++++++++++++++++++++-------------------- packages/defaults.yaml | 1 + pillar.example | 15 ++++++- 5 files changed, 64 insertions(+), 50 deletions(-) diff --git a/.kitchen.yml b/.kitchen.yml index b2ebd40..6f4f3f4 100644 --- a/.kitchen.yml +++ b/.kitchen.yml @@ -51,7 +51,8 @@ verifier: name: inspec sudo: false # cli, documentation, html, progress, json, json-min, json-rspec, junit - reporter: cli + reporter: + - cli inspec_tests: - path: test/integration/default diff --git a/README.rst b/README.rst index 5652560..fa0c958 100644 --- a/README.rst +++ b/README.rst @@ -130,14 +130,12 @@ You can specify: ``packages.archives`` ------------------- -'Archive file` handler for common 'download' and 'checksum' states; extraction state based on `format` value. +'Archive file` handler for common 'download' and 'checksum' states. All formats recognized by `salt.states.archive.extracted` (tar, rar, zip, etc) will be extracted. Alternatively `raw` formats are supported (`raw`, `bin`,) for standard and binary executable files. * ``wanted`` archive package software, which will be installed by extraction. * ``unwanted`` archive package software, which are uninstalled by directory removal. * ``required archive packages`` on which any of the ``wanted`` items depend on. Optional. -.. note:: Supports `tar` formats that `salt.states.archive.extracted` understands (tar, rar, zip, etc). The `packages.archives` state can be extended. - ``packages.snaps`` ----------------- diff --git a/packages/archives.sls b/packages/archives.sls index 9c7632e..af1c723 100644 --- a/packages/archives.sls +++ b/packages/archives.sls @@ -24,83 +24,86 @@ packages-archive-unwanted-{{ file_or_directory }}: # wanted 'archive' software {% for package, archive in wanted_archives.items() %} + {%- set archivename = archive.dl.source.split('/')[-1] %} -packages-archive-wanted-remove-prev-{{ package }}: - file.absent: - - name: {{ packages.tmpdir }}/{{ package }} - - require_in: - - packages-archive-wanted-extract-{{ package }}-directory - -packages-archive-wanted-extract-{{ package }}-directory: +packages-archive-wanted-target-{{ package }}-directory: file.directory: - names: - {{ packages.tmpdir }}/tmp - {{ archive.dest }} - - user: {{ 'root' if "user" not in archive else archive.user }} - - mode: {{ '0755' if "mode" not in archive else archive.mode }} + - user: {{ 'root' if 'user' not in archive else archive.user }} + - mode: {{ '0755' if 'mode' not in archive else archive.mode }} - makedirs: True - require_in: - - cmd: packages-archive-wanted-download-{{ package }} + - packages-archive-wanted-download-{{ package }} + + {%- if 'format' in archive.dl.format and archive.dl.format in packages.archives.types %} + +packages-archive-wanted-remove-prev-{{ package }}: + file.absent: + - name: {{ packages.tmpdir }}/{{ archivename }} + - require_in: + - packages-archive-wanted-target-{{ package }}-directory packages-archive-wanted-download-{{ package }}: cmd.run: - - name: curl -s -L -o {{ packages.tmpdir }}/{{ package }} {{ archive.dl.source }} - - unless: test -f {{ packages.tmpdir }}/{{ package }} + - name: curl -s -L -o {{ packages.tmpdir }}/{{ archivename }} {{ archive.dl.source }} + - unless: test -f {{ packages.tmpdir }}/{{ archivename }}/ - {%- if "hashsum" in archive.dl and archive.dl.hashsum %} - {# refer to https://github.com/saltstack/salt/pull/41914 #} + {%- if 'hashsum' in archive.dl and archive.dl.hashsum %} + {# see https://github.com/saltstack/salt/pull/41914 #} packages-archive-wanted-{{ package }}-check-hashsum: module.run: - name: file.check_hash - - path: {{ packages.tmpdir }}/{{ package }} + - path: {{ packages.tmpdir }}/{{ archivename }} - file_hash: {{ archive.dl.hashsum }} - require: - - cmd: packages-archive-wanted-download-{{ package }} + - packages-archive-wanted-download-{{ package }} - require_in: - archive: packages-archive-wanted-install-{{ package }} - cmd.run: - - name: rm {{ packages.tmpdir }}/{{ package }} - - onfail: - - module: packages-archive-wanted-{{ package }}-check-hashsum - {%- endif %} -packages-archive-wanted-install-{{ package }}: - {% if archive.dl.format|trim|lower in ('tar','zip', 'rar',) %} + {%- endif %} +packages-archive-wanted-install-{{ package }}: archive.extracted: - - source: file://{{ packages.tmpdir }}/{{ package }} - - name: {{ archive.dest }} + - source: file://{{ packages.tmpdir }}/{{ archivename }} + - name: {{ archive.dest }}/ - archive_format: {{ archive.dl.format }} - {%- if 'hashurl' in archive.dl and archive.dl.hashurl %} + {%- if 'hashurl' in archive.dl and archive.dl.hashurl %} - source_hash: {{ archive.dl.hashurl }} - {%- endif %} - {%- if 'options' in archive and archive.options %} + {%- endif %} + {%- if 'options' in archive and archive.options %} - options: {{ archive.options }} - - enforce_toplevel: {{ 'False' if "strip-components" in archive.options else 'True' }} - {%- endif %} + - enforce_toplevel: {{ 'False' if 'strip-components' in archive.options else 'True' }} + {%- endif %} - unless: test -d {{ archive.dest }} - cmd.run: - - name: rm {{ packages.tmpdir }}/{{ package }} - - onfail: - - archive: packages-archive-wanted-install-{{ package }} - - {% else %} - - test.show_notification: - - text: | - The value of "packages.archives.wanted.{{ package }}.dl.format' is unsupported (skipping {{ package }}). - - {% endif %} - - onchanges: + - require: - packages-archive-wanted-download-{{ package }} + - module: packages-archive-wanted-{{ package }}-check-hashsum - require_in: - packages-archive-wanted-cleanup-{{ package }} packages-archive-wanted-cleanup-{{ package }}: file.absent: - - name: {{ packages.tmpdir }}/{{ package }} + - name: {{ packages.tmpdir }}/{{ archivename }} - onchanges: - packages-archive-wanted-install-{{ package }} + {%- else %} + +packages-archive-wanted-download-{{ package }}: + file.managed: + - name: {{ archive.dest }}/{{ archivename }} + - source: {{ archive.dl.source }} + - mode: {{ '0755' if archive.dl.format in ('bin',) else '0644' if 'mode' not in archive else archive.mode }} + - user: {{ 'root' if 'user' not in archive else archive.user }} + - makedirs: True + {%- if 'hashsum' in archive.dl and archive.dl.hashsum %} + - source_hash: {{ archive.dl.hashsum }} + {%- else %} + - skip_verify: True + {%- endif %} + + {% endif %} {%- endfor %} diff --git a/packages/defaults.yaml b/packages/defaults.yaml index 9fc8a08..21ec754 100644 --- a/packages/defaults.yaml +++ b/packages/defaults.yaml @@ -34,6 +34,7 @@ packages: states: [] pkgs: [] archives: + types: ('tar','zip', 'rar',) wanted: {} #note: dict unwanted: [] required: diff --git a/pillar.example b/pillar.example index 11ef330..70c24ed 100644 --- a/pillar.example +++ b/pillar.example @@ -49,7 +49,7 @@ packages: archives: wanted: terminator: - dest: /usr/local/terminator + dest: /usr/local/terminator/ options: '--strip-components=1' #recommended option, but beware tarbombs dl: format: tar @@ -57,13 +57,24 @@ packages: #hashurl: https://launchpad.net/terminator/gtk3/1.91/+download/terminator-1.91.tar.gz/+md5 hashsum: md5=2eed999d7a41f2e18eaa511bbbf80f58 phantomjs: - dest: /usr/local/src #beware tarbombs + dest: /usr/local/src/ #beware tarbombs user: root mode: '0700' dl: format: tar source: https://bitbucket.org/ariya/phantomjs/downloads/phantomjs-2.1.1-linux-x86_64.tar.bz2 hashsum: md5=1c947d57fce2f21ce0b43fe2ed7cd361 + blockbox: + dest: /usr/local/src/ + dl: + format: raw + source: https://raw.githubusercontent.com/openstack/cinder/master/contrib/block-box/docker-compose.yml + hashsum: 1751f8e4f6b4cddd8c4843a0f4473274 + kubectl: + dest: /usr/local/bin + dl: + format: bin + source: https://storage.googleapis.com/kubernetes-release/release/v1.12.0/bin/darwin/amd64/kubectl unwanted: - /usr/local/boring_archive_software