From 030aeddaecbd5f8271887ca7618cf63b8f87062c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Igor=20Gali=C4=87?= Date: Sun, 3 Feb 2019 13:44:50 +0100 Subject: [PATCH 01/29] crude design attempt for a puppet-apply provisioning plugin this is mostly copied from the iX plugin and renames / removes a bunch of things we don't need. There's still a lot of confusion wrt the naming. We also need to generalize the plugin so we'll be able to work with both `puppet apply` and `puppet agent` --- libioc/Provisioning/puppet.py | 209 ++++++++++++++++++++++++++++++++++ 1 file changed, 209 insertions(+) create mode 100644 libioc/Provisioning/puppet.py diff --git a/libioc/Provisioning/puppet.py b/libioc/Provisioning/puppet.py new file mode 100644 index 00000000..178ed5be --- /dev/null +++ b/libioc/Provisioning/puppet.py @@ -0,0 +1,209 @@ +# Copyright (c) 2017-2019, Stefan Grönke, Igor Galić +# Copyright (c) 2014-2018, ioc +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted providing that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR +# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING +# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +"""ioc provisioner for use with `puppet apply`.""" +import typing +import os.path +import json +import urllib.error +import urllib.request +import libzfs + +import git + +import libioc.errors +import libioc.events +import libioc.Pkg +import libioc.Provisioning + + +class ControlRepoUnavailableError(libioc.errors.IocException): + """Raised when the puppet control-repo is not available.""" + + def __init__( + self, + url: str, + reason: str, + logger: typing.Optional['libioc.Logger.Logger']=None + ) -> None: + msg = f"Puppet control-repo '{url}' is not available: {reason}" + libioc.errors.IocException.__init__( + self, + message=msg, + logger=logger + ) + + +class ControlRepoDefinition(dict): + """Puppet control-repo definition.""" + + _url: str + _name: str + _pkgs: typing.List[str] + + def __init__( + self, + name: str, + url: str, + logger: 'libioc.Logger.Logger' + ) -> None: + self.logger = logger + self.url = url + self.name = name + + _pkgs = ['puppet6'] # make this a Global Varialbe + if not (url.startswith('file://') or url.startswith('/')): + _pkgs += 'rubygem-r10k' + + @property + def url(self) -> str: + """Return the Puppet Control-Repo URL.""" + return str(self._url) + + @property.setter + def url(self) -> str: + """Set the Puppet Control-Repo URL.""" + return str(self._url) + + @property + def name(self, value: str) -> str: + """Return the unique name for this Puppet Control-Repo URL.""" + return self._name + + @property.setter + def name(self, value: str) -> None: + """Set a unique name for this Puppet Control-Repo URL.""" + self._name = value + + @property + def pkgs(self, value: str) -> typing.List[str]: + """Return list of packages required for this Provisioning method.""" + return self._pkgs + + #@property.setter + #def pkgs(self, value: str) -> None: + # """Set (list) of additional packagess required for this Puppet Control-Repo URL.""" + # self._pkgs += value + + +def provision( + self: 'libioc.Provisioning.Prototype', + event_scope: typing.Optional['libioc.events.Scope']=None +) -> typing.Generator['libioc.events.IocEvent', None, None]: + """ + Provision the jail with Puppet apply using the supplied control-repo. + + The repo can either be a filesystem path, or a http[s]/git URL. + If the repo is a filesystem path, it will be mounted appropriately. + If the repo is a URL, it will be setup with `r10k`. + + ioc set provisioning.method=puppet provisioning.source=http://example.com/my/puppet-env myjail + + """ + events = libioc.events + jailProvisioningEvent = events.JailProvisioning( + jail=self.jail, + event_scope=event_scope + ) + yield jailProvisioningEvent.begin() + _scope = jailProvisioningEvent.scope + jailProvisioningAssetDownloadEvent = events.JailProvisioningAssetDownload( + jail=self.jail, + event_scope=_scope + ) + + # download / mount provisioning assets + try: + yield jailProvisioningAssetDownloadEvent.begin() + pluginDefinition = ControlRepoDefinition( + self.source, + logger=self.jail.logger + ) + yield jailProvisioningAssetDownloadEvent.end() + except Exception as e: + yield jailProvisioningAssetDownloadEvent.fail(e) + raise e + + if not (self.source.startswith('file://') or self.source.startswith('/')): + # clone control repo + controlrepo_dataset_name = f"{self.jail.dataset.name}/puppet" + controlrepo_dataset = __get_empty_dataset( + control_repo_dataset_name, self.jail.zfs) + + git.Repo.clone_from( + ControlRepoDefinition["url"], + plugin_dataset.mountpoint + ) + + self.jail.fstab.new_line( + source=plugin_dataset.mountpoint, + destination="/.puppet", + options="ro", + auto_create_destination=True, + replace=True + ) + self.jail.fstab.save() + + if "pkgs" in controlRepoDefinition.keys(): + pkg_packages = list(controlRepoDefinition["pkgs"]) + else: + pkg_packages = [] + + try: + pkg = libioc.Pkg.Pkg( + logger=self.jail.logger, + zfs=self.jail.zfs, + host=self.jail.host + ) + + if os.path.isfile(f"{plugin_dataset.mountpoint}/post_install.sh"): + postinstall = ["/.puppet/post_install.sh"] + else: + postinstall = [] + + yield from pkg.fetch_and_install( + jail=self.jail, + packages=pkg_packages, + postinstall=postinstall + ) + except Exception as e: + yield jailProvisioningEvent.fail(e) + raise e + + +def __get_empty_dataset( + dataset_name: str, + zfs: 'libioc.ZFS.ZFS' +) -> libzfs.ZFSDataset: + try: + dataset = zfs.get_dataset(dataset_name) + except libzfs.ZFSException: + dataset = None + pass + if dataset is not None: + dataset.umount() + zfs.delete_dataset_recursive(dataset) + + output: libzfs.ZFSDataset = zfs.get_or_create_dataset(dataset_name) + return output From e15d43e3ecbf1453861782141516905aa64746a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Igor=20Gali=C4=87?= Date: Sun, 3 Feb 2019 14:21:20 +0100 Subject: [PATCH 02/29] remove iocage (c) from copy/pasted code --- libioc/Provisioning/puppet.py | 1 - 1 file changed, 1 deletion(-) diff --git a/libioc/Provisioning/puppet.py b/libioc/Provisioning/puppet.py index 178ed5be..2d27a807 100644 --- a/libioc/Provisioning/puppet.py +++ b/libioc/Provisioning/puppet.py @@ -1,5 +1,4 @@ # Copyright (c) 2017-2019, Stefan Grönke, Igor Galić -# Copyright (c) 2014-2018, ioc # All rights reserved. # # Redistribution and use in source and binary forms, with or without From b979db85ad66ad4ecbbe28b6e7b5f33c00013989 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Gr=C3=B6nke?= Date: Sun, 3 Feb 2019 14:45:03 +0100 Subject: [PATCH 03/29] add puppet to available provisioning modules --- libioc/Provisioning/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libioc/Provisioning/__init__.py b/libioc/Provisioning/__init__.py index 80b67a02..e7bd1f56 100644 --- a/libioc/Provisioning/__init__.py +++ b/libioc/Provisioning/__init__.py @@ -84,7 +84,8 @@ def __available_provisioning_modules( self ) -> typing.Dict[str, Prototype]: return dict( - ix=libioc.Provisioning.ix + ix=libioc.Provisioning.ix, + puppet=libioc.Provisioning.puppet ) @property From 4cb9e05aab4d93b3628b070c0005bdb42bb074fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Gr=C3=B6nke?= Date: Sun, 3 Feb 2019 18:03:16 +0100 Subject: [PATCH 04/29] break cli example in puppet provisioner in multiple lines --- libioc/Provisioning/puppet.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/libioc/Provisioning/puppet.py b/libioc/Provisioning/puppet.py index 2d27a807..cbc31b12 100644 --- a/libioc/Provisioning/puppet.py +++ b/libioc/Provisioning/puppet.py @@ -117,7 +117,10 @@ def provision( If the repo is a filesystem path, it will be mounted appropriately. If the repo is a URL, it will be setup with `r10k`. - ioc set provisioning.method=puppet provisioning.source=http://example.com/my/puppet-env myjail + ioc set \ + provisioning.method=puppet \ + provisioning.source=http://example.com/my/puppet-env \ + myjail """ events = libioc.events From c5063d68a6978b52c7d8420ce5ff79edab43f7e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Igor=20Gali=C4=87?= Date: Sun, 3 Feb 2019 21:38:21 +0100 Subject: [PATCH 05/29] pkgs is a property and needs to be accessed by . address review by @gronke --- libioc/Provisioning/puppet.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libioc/Provisioning/puppet.py b/libioc/Provisioning/puppet.py index cbc31b12..322b8245 100644 --- a/libioc/Provisioning/puppet.py +++ b/libioc/Provisioning/puppet.py @@ -168,7 +168,7 @@ def provision( self.jail.fstab.save() if "pkgs" in controlRepoDefinition.keys(): - pkg_packages = list(controlRepoDefinition["pkgs"]) + pkg_packages = list(controlRepoDefinition.pkgs) else: pkg_packages = [] From 312ed37a84b54c908bdbcf77489e7757ee5be78a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Igor=20Gali=C4=87?= Date: Mon, 4 Feb 2019 17:11:27 +0100 Subject: [PATCH 06/29] be consistent with naming: use pluginDefinition as variable name regardless of what type of plugin it was created from --- libioc/Provisioning/puppet.py | 35 +++++++++++++++++++++++++++-------- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/libioc/Provisioning/puppet.py b/libioc/Provisioning/puppet.py index 322b8245..f9d46792 100644 --- a/libioc/Provisioning/puppet.py +++ b/libioc/Provisioning/puppet.py @@ -100,12 +100,26 @@ def pkgs(self, value: str) -> typing.List[str]: """Return list of packages required for this Provisioning method.""" return self._pkgs + def generate_postinstall(self) -> typing.List[str]: + if self.remote: + # write /usr/local/etc/r10k/r10k.yaml with + # --- + # :sources: + # puppet: + # basedir: /usr/local/etc/puppet/environments + # remote: {self.url} + + # run r10k -p + pass + + # run puppet apply {debug} {manifest} + return [''] + #@property.setter #def pkgs(self, value: str) -> None: # """Set (list) of additional packagess required for this Puppet Control-Repo URL.""" # self._pkgs += value - def provision( self: 'libioc.Provisioning.Prototype', event_scope: typing.Optional['libioc.events.Scope']=None @@ -139,7 +153,8 @@ def provision( try: yield jailProvisioningAssetDownloadEvent.begin() pluginDefinition = ControlRepoDefinition( - self.source, + name=self.source.name, + url=self.source, logger=self.jail.logger ) yield jailProvisioningAssetDownloadEvent.end() @@ -150,25 +165,29 @@ def provision( if not (self.source.startswith('file://') or self.source.startswith('/')): # clone control repo controlrepo_dataset_name = f"{self.jail.dataset.name}/puppet" - controlrepo_dataset = __get_empty_dataset( + controlrepo_dataset = self.zfs.get_or_create_dataset( control_repo_dataset_name, self.jail.zfs) git.Repo.clone_from( - ControlRepoDefinition["url"], + pluginDefinition.url, plugin_dataset.mountpoint ) + mount_source = plugin_dataset.mountpoint + else: + mount_source = source + self.jail.fstab.new_line( - source=plugin_dataset.mountpoint, - destination="/.puppet", + source=mount_source, + destination="/usr/local/etc/puppet", options="ro", auto_create_destination=True, replace=True ) self.jail.fstab.save() - if "pkgs" in controlRepoDefinition.keys(): - pkg_packages = list(controlRepoDefinition.pkgs) + if "pkgs" in pluginDefinition.keys(): + pkg_packages = list(pluginDefinition.pkgs) else: pkg_packages = [] From 0ea62f41d6d4a37db691211510ebeaa64fc548ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Igor=20Gali=C4=87?= Date: Mon, 4 Feb 2019 17:12:15 +0100 Subject: [PATCH 07/29] add local & remote property to our plugin definition use these for generate_postscript which, so far, is still just a hollow ghost. --- libioc/Provisioning/puppet.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/libioc/Provisioning/puppet.py b/libioc/Provisioning/puppet.py index f9d46792..51f86ea4 100644 --- a/libioc/Provisioning/puppet.py +++ b/libioc/Provisioning/puppet.py @@ -75,6 +75,16 @@ def __init__( if not (url.startswith('file://') or url.startswith('/')): _pkgs += 'rubygem-r10k' + @property + def local(self) -> bool: + if not (self.url.startswith('file://') or self.url.startswith('/')): + return False + return True + + @property + def remote(self) -> bool: + return not self.local + @property def url(self) -> str: """Return the Puppet Control-Repo URL.""" @@ -134,6 +144,7 @@ def provision( ioc set \ provisioning.method=puppet \ provisioning.source=http://example.com/my/puppet-env \ + provisioning.source.name=my-puppet-env \ myjail """ @@ -198,10 +209,8 @@ def provision( host=self.jail.host ) - if os.path.isfile(f"{plugin_dataset.mountpoint}/post_install.sh"): - postinstall = ["/.puppet/post_install.sh"] - else: - postinstall = [] + postinstall_scripts = pluginDefinition.generate_postinstall() + # write postinstall yield from pkg.fetch_and_install( jail=self.jail, From c6a45c5fabc3a8e5a22630923d05106ee8ec9d47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Igor=20Gali=C4=87?= Date: Mon, 4 Feb 2019 18:18:06 +0100 Subject: [PATCH 08/29] puppet: use persistent dataset --- libioc/Provisioning/puppet.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/libioc/Provisioning/puppet.py b/libioc/Provisioning/puppet.py index 51f86ea4..ebfba690 100644 --- a/libioc/Provisioning/puppet.py +++ b/libioc/Provisioning/puppet.py @@ -175,9 +175,10 @@ def provision( if not (self.source.startswith('file://') or self.source.startswith('/')): # clone control repo - controlrepo_dataset_name = f"{self.jail.dataset.name}/puppet" - controlrepo_dataset = self.zfs.get_or_create_dataset( - control_repo_dataset_name, self.jail.zfs) + plugin_dataset_name = f"{self.jail.dataset.name}/puppet" + plugin_dataset = self.zfs.get_or_create_dataset( + plugin_dataset_name + ) git.Repo.clone_from( pluginDefinition.url, From 3e925a3b56973fb58977db5ebbefe35d07d49dd4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Igor=20Gali=C4=87?= Date: Mon, 4 Feb 2019 18:42:20 +0100 Subject: [PATCH 09/29] remove unused copy/pasted function --- libioc/Provisioning/puppet.py | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/libioc/Provisioning/puppet.py b/libioc/Provisioning/puppet.py index ebfba690..fcc6382d 100644 --- a/libioc/Provisioning/puppet.py +++ b/libioc/Provisioning/puppet.py @@ -222,19 +222,3 @@ def provision( yield jailProvisioningEvent.fail(e) raise e - -def __get_empty_dataset( - dataset_name: str, - zfs: 'libioc.ZFS.ZFS' -) -> libzfs.ZFSDataset: - try: - dataset = zfs.get_dataset(dataset_name) - except libzfs.ZFSException: - dataset = None - pass - if dataset is not None: - dataset.umount() - zfs.delete_dataset_recursive(dataset) - - output: libzfs.ZFSDataset = zfs.get_or_create_dataset(dataset_name) - return output From 8fa883d1f9e429192f24cde4a64a812540ed7c80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Igor=20Gali=C4=87?= Date: Mon, 4 Feb 2019 18:53:19 +0100 Subject: [PATCH 10/29] only clone the repo if it doesnt already exist let r10k handle the rest! and, to let r10k handle the rest, we need to mount the dataset as rw --- libioc/Provisioning/puppet.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/libioc/Provisioning/puppet.py b/libioc/Provisioning/puppet.py index fcc6382d..a22386ca 100644 --- a/libioc/Provisioning/puppet.py +++ b/libioc/Provisioning/puppet.py @@ -25,8 +25,6 @@ import typing import os.path import json -import urllib.error -import urllib.request import libzfs import git @@ -106,11 +104,12 @@ def name(self, value: str) -> None: self._name = value @property - def pkgs(self, value: str) -> typing.List[str]: + def pkgs(self) -> typing.List[str]: """Return list of packages required for this Provisioning method.""" return self._pkgs def generate_postinstall(self) -> typing.List[str]: + """Return list of strings representing our postinstall""" if self.remote: # write /usr/local/etc/r10k/r10k.yaml with # --- @@ -130,11 +129,12 @@ def generate_postinstall(self) -> typing.List[str]: # """Set (list) of additional packagess required for this Puppet Control-Repo URL.""" # self._pkgs += value + def provision( self: 'libioc.Provisioning.Prototype', event_scope: typing.Optional['libioc.events.Scope']=None ) -> typing.Generator['libioc.events.IocEvent', None, None]: - """ + r""" Provision the jail with Puppet apply using the supplied control-repo. The repo can either be a filesystem path, or a http[s]/git URL. @@ -180,19 +180,21 @@ def provision( plugin_dataset_name ) - git.Repo.clone_from( - pluginDefinition.url, - plugin_dataset.mountpoint - ) + if not os.path.isdir(plugin_dataset_name): + # only clone if it doesn't already exist + git.Repo.clone_from( + pluginDefinition.url, + plugin_dataset.mountpoint + ) mount_source = plugin_dataset.mountpoint else: - mount_source = source + mount_source = self.source self.jail.fstab.new_line( source=mount_source, destination="/usr/local/etc/puppet", - options="ro", + options="rw", auto_create_destination=True, replace=True ) From ca1995820d4c5b48b00b0b43f45ccf171fe6a046 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Igor=20Gali=C4=87?= Date: Mon, 4 Feb 2019 19:01:29 +0100 Subject: [PATCH 11/29] only mount plugin dataset as rw when it needs to be rw --- libioc/Provisioning/puppet.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/libioc/Provisioning/puppet.py b/libioc/Provisioning/puppet.py index a22386ca..41943c50 100644 --- a/libioc/Provisioning/puppet.py +++ b/libioc/Provisioning/puppet.py @@ -174,7 +174,7 @@ def provision( raise e if not (self.source.startswith('file://') or self.source.startswith('/')): - # clone control repo + mode = 'rw' # we'll need to run r10k here.. plugin_dataset_name = f"{self.jail.dataset.name}/puppet" plugin_dataset = self.zfs.get_or_create_dataset( plugin_dataset_name @@ -189,12 +189,13 @@ def provision( mount_source = plugin_dataset.mountpoint else: + mode = 'ro' mount_source = self.source self.jail.fstab.new_line( source=mount_source, destination="/usr/local/etc/puppet", - options="rw", + options=mode, auto_create_destination=True, replace=True ) From 5a266d1a77454d42446ebb88843f407d2d7bb85a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Igor=20Gali=C4=87?= Date: Mon, 4 Feb 2019 23:10:06 +0100 Subject: [PATCH 12/29] write postinstall (provisioning) script --- libioc/Provisioning/puppet.py | 46 +++++++++++++++++++---------------- 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/libioc/Provisioning/puppet.py b/libioc/Provisioning/puppet.py index 41943c50..d968314b 100644 --- a/libioc/Provisioning/puppet.py +++ b/libioc/Provisioning/puppet.py @@ -24,8 +24,6 @@ """ioc provisioner for use with `puppet apply`.""" import typing import os.path -import json -import libzfs import git @@ -108,26 +106,30 @@ def pkgs(self) -> typing.List[str]: """Return list of packages required for this Provisioning method.""" return self._pkgs - def generate_postinstall(self) -> typing.List[str]: - """Return list of strings representing our postinstall""" + def generate_postinstall(self) -> str: + """Return list of strings representing our postinstall.""" + postinstall = """#!/bin/sh + set -eu + + """ + if self.remote: - # write /usr/local/etc/r10k/r10k.yaml with - # --- - # :sources: - # puppet: - # basedir: /usr/local/etc/puppet/environments - # remote: {self.url} + postinstall += """cat > /usr/local/etc/r10k/r10k.yml EOF - # run r10k -p - pass + r10k deploy environment -p - # run puppet apply {debug} {manifest} - return [''] + """ - #@property.setter - #def pkgs(self, value: str) -> None: - # """Set (list) of additional packagess required for this Puppet Control-Repo URL.""" - # self._pkgs += value + postinstall += """ + puppet apply /usr/local/etc/puppet/environments/manifests/site.pp + """ + return postinstall def provision( @@ -173,7 +175,7 @@ def provision( yield jailProvisioningAssetDownloadEvent.fail(e) raise e - if not (self.source.startswith('file://') or self.source.startswith('/')): + if pluginDefinition.remote: mode = 'rw' # we'll need to run r10k here.. plugin_dataset_name = f"{self.jail.dataset.name}/puppet" plugin_dataset = self.zfs.get_or_create_dataset( @@ -213,8 +215,10 @@ def provision( host=self.jail.host ) - postinstall_scripts = pluginDefinition.generate_postinstall() - # write postinstall + postinstall_script = pluginDefinition.generate_postinstall() + postinstall = "{self.jail.abspath}/launch-scripts/provision.sh" + with open(postinstall, 'w') as f: + f.write(postinstall_script) yield from pkg.fetch_and_install( jail=self.jail, From 35359c7e3c1180e1843ef54500b2fc47e3540c0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Igor=20Gali=C4=87?= Date: Mon, 4 Feb 2019 23:17:06 +0100 Subject: [PATCH 13/29] fix flake8 warnings --- libioc/Provisioning/puppet.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/libioc/Provisioning/puppet.py b/libioc/Provisioning/puppet.py index d968314b..ade510fb 100644 --- a/libioc/Provisioning/puppet.py +++ b/libioc/Provisioning/puppet.py @@ -73,12 +73,14 @@ def __init__( @property def local(self) -> bool: + """Return whether this control repo resides locally.""" if not (self.url.startswith('file://') or self.url.startswith('/')): return False return True @property def remote(self) -> bool: + """Return whether this control repo resides locally.""" return not self.local @property @@ -86,17 +88,17 @@ def url(self) -> str: """Return the Puppet Control-Repo URL.""" return str(self._url) - @property.setter - def url(self) -> str: + @url.setter + def url(self, value: str) -> None: """Set the Puppet Control-Repo URL.""" - return str(self._url) + self._url = value @property - def name(self, value: str) -> str: + def name(self) -> str: """Return the unique name for this Puppet Control-Repo URL.""" return self._name - @property.setter + @name.setter def name(self, value: str) -> None: """Set a unique name for this Puppet Control-Repo URL.""" self._name = value From e19087ffbbf9cecab1dc57c12ed8b72424df0361 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Gr=C3=B6nke?= Date: Tue, 5 Feb 2019 14:15:51 +0000 Subject: [PATCH 14/29] fix scope and name in puppet provisioning --- libioc/Provisioning/__init__.py | 1 + libioc/Provisioning/puppet.py | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/libioc/Provisioning/__init__.py b/libioc/Provisioning/__init__.py index e7bd1f56..fb79ee15 100644 --- a/libioc/Provisioning/__init__.py +++ b/libioc/Provisioning/__init__.py @@ -28,6 +28,7 @@ import libioc.errors import libioc.helpers import libioc.Provisioning.ix +import libioc.Provisioning.puppet class Prototype: diff --git a/libioc/Provisioning/puppet.py b/libioc/Provisioning/puppet.py index ade510fb..7fb66298 100644 --- a/libioc/Provisioning/puppet.py +++ b/libioc/Provisioning/puppet.py @@ -148,27 +148,27 @@ def provision( ioc set \ provisioning.method=puppet \ provisioning.source=http://example.com/my/puppet-env \ - provisioning.source.name=my-puppet-env \ + provisioning.name=my-puppet-env \ myjail """ events = libioc.events jailProvisioningEvent = events.JailProvisioning( jail=self.jail, - event_scope=event_scope + scope=event_scope ) yield jailProvisioningEvent.begin() _scope = jailProvisioningEvent.scope jailProvisioningAssetDownloadEvent = events.JailProvisioningAssetDownload( jail=self.jail, - event_scope=_scope + scope=_scope ) # download / mount provisioning assets try: yield jailProvisioningAssetDownloadEvent.begin() pluginDefinition = ControlRepoDefinition( - name=self.source.name, + name=self.name, url=self.source, logger=self.jail.logger ) From f48a1b59fd8f4b5adf1871d03bc6847589b5ffc5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Gr=C3=B6nke?= Date: Tue, 5 Feb 2019 21:15:48 +0000 Subject: [PATCH 15/29] puppet provisioning uses urllib.parse --- libioc/Provisioning/puppet.py | 48 +++++++++++++++++++---------------- 1 file changed, 26 insertions(+), 22 deletions(-) diff --git a/libioc/Provisioning/puppet.py b/libioc/Provisioning/puppet.py index 7fb66298..4f06464f 100644 --- a/libioc/Provisioning/puppet.py +++ b/libioc/Provisioning/puppet.py @@ -24,6 +24,7 @@ """ioc provisioner for use with `puppet apply`.""" import typing import os.path +import urllib.parse import git @@ -54,27 +55,28 @@ class ControlRepoDefinition(dict): """Puppet control-repo definition.""" _url: str - _name: str _pkgs: typing.List[str] def __init__( self, - name: str, - url: str, + url: urllib.parse.DefragResult, logger: 'libioc.Logger.Logger' ) -> None: self.logger = logger + + if isinstance(url, urllib.parse.ParseResult) is False: + raise TypeError("Source must be an URL") self.url = url - self.name = name - _pkgs = ['puppet6'] # make this a Global Varialbe - if not (url.startswith('file://') or url.startswith('/')): - _pkgs += 'rubygem-r10k' + self._pkgs = ['puppet6'] # make this a Global Varialbe + if not (self.url.startswith('file://') or self.url.startswith('/')): + self._pkgs += 'rubygem-r10k' @property def local(self) -> bool: """Return whether this control repo resides locally.""" - if not (self.url.startswith('file://') or self.url.startswith('/')): + _url = self.url + if not (_url.startswith('file://') or _url.startswith('/')): return False return True @@ -86,22 +88,26 @@ def remote(self) -> bool: @property def url(self) -> str: """Return the Puppet Control-Repo URL.""" - return str(self._url) + return self._url @url.setter - def url(self, value: str) -> None: + def url(self, value: typing.Union[urllib.parse.ParseResult, str]) -> None: """Set the Puppet Control-Repo URL.""" - self._url = value + _url: urllib.parse.ParseResult + if isinstance(value, urllib.parse.ParseResult) is True: + _url = typing.cast(urllib.parse.ParseResult, value) + elif isinstance(value, str) is True: + _url = urllib.parse.urlparse( + str(value), + allow_fragments=False + ) + else: + raise TypeError("URL must be urllib.parse.ParseResult or string") - @property - def name(self) -> str: - """Return the unique name for this Puppet Control-Repo URL.""" - return self._name + if _url.fragment != "": + raise ValueError("URL may not contain fragment") - @name.setter - def name(self, value: str) -> None: - """Set a unique name for this Puppet Control-Repo URL.""" - self._name = value + self._url = _url.geturl() @property def pkgs(self) -> typing.List[str]: @@ -148,7 +154,6 @@ def provision( ioc set \ provisioning.method=puppet \ provisioning.source=http://example.com/my/puppet-env \ - provisioning.name=my-puppet-env \ myjail """ @@ -168,8 +173,7 @@ def provision( try: yield jailProvisioningAssetDownloadEvent.begin() pluginDefinition = ControlRepoDefinition( - name=self.name, - url=self.source, + url=urllib.parse.urlparse(self.source).geturl(), logger=self.jail.logger ) yield jailProvisioningAssetDownloadEvent.end() From 87e3a43483acec03d6d20ee16937256d47dfc441 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Igor=20Gali=C4=87?= Date: Wed, 6 Feb 2019 13:25:37 +0100 Subject: [PATCH 16/29] rename url to source and allow it to be AbsolutePath This fixes a regression with previous functionality! --- libioc/Provisioning/puppet.py | 59 ++++++++++++++++++++--------------- 1 file changed, 33 insertions(+), 26 deletions(-) diff --git a/libioc/Provisioning/puppet.py b/libioc/Provisioning/puppet.py index 4f06464f..9b6f1608 100644 --- a/libioc/Provisioning/puppet.py +++ b/libioc/Provisioning/puppet.py @@ -54,29 +54,33 @@ def __init__( class ControlRepoDefinition(dict): """Puppet control-repo definition.""" - _url: str + _source: str _pkgs: typing.List[str] def __init__( self, - url: urllib.parse.DefragResult, + source: typing.Union[ + urllib.parse.DefragResult, + libioc.Types.AbsolutePath + ], logger: 'libioc.Logger.Logger' ) -> None: self.logger = logger - if isinstance(url, urllib.parse.ParseResult) is False: - raise TypeError("Source must be an URL") - self.url = url + if isinstance(source, libioc.Types.AbsolutePath) is False \ + and isinstance(source, urllib.parse.ParseResult) is False: + raise TypeError("Source must be an URL or an absolute path") + self.source = source self._pkgs = ['puppet6'] # make this a Global Varialbe - if not (self.url.startswith('file://') or self.url.startswith('/')): + if isinstance(source, libioc.Types.AbsolutePath) is False: self._pkgs += 'rubygem-r10k' @property def local(self) -> bool: """Return whether this control repo resides locally.""" - _url = self.url - if not (_url.startswith('file://') or _url.startswith('/')): + _source = self.source + if isinstance(_source, libioc.Types.AbsolutePath) is False: return False return True @@ -86,28 +90,30 @@ def remote(self) -> bool: return not self.local @property - def url(self) -> str: + def source(self) -> str: """Return the Puppet Control-Repo URL.""" - return self._url + return self._source - @url.setter - def url(self, value: typing.Union[urllib.parse.ParseResult, str]) -> None: + @source.setter + def source(self, value: typing.Union[ + urllib.parse.ParseResult, + libioc.Types.AbsolutePath + ]) -> None: """Set the Puppet Control-Repo URL.""" - _url: urllib.parse.ParseResult + _source: urllib.parse.ParseResult if isinstance(value, urllib.parse.ParseResult) is True: - _url = typing.cast(urllib.parse.ParseResult, value) - elif isinstance(value, str) is True: - _url = urllib.parse.urlparse( - str(value), - allow_fragments=False - ) - else: - raise TypeError("URL must be urllib.parse.ParseResult or string") + _source = typing.cast(urllib.parse.ParseResult, value) + if _source.fragment != "": + raise ValueError("URL may not contain fragment") - if _url.fragment != "": - raise ValueError("URL may not contain fragment") + self._source = _source.geturl() + elif isinstance(value, libioc.Types.AbsolutePath) is True: + _source = value + else: + raise TypeError( + "Source must be urllib.parse.ParseResult or absolute path" + ) - self._url = _url.geturl() @property def pkgs(self) -> typing.List[str]: @@ -116,6 +122,7 @@ def pkgs(self) -> typing.List[str]: def generate_postinstall(self) -> str: """Return list of strings representing our postinstall.""" + basedir = "/usr/local/etc/puppet/environments" postinstall = """#!/bin/sh set -eu @@ -126,8 +133,8 @@ def generate_postinstall(self) -> str: --- :source: puppet: - basedir: /usr/local/etc/puppet/environments - remote: {self.url} + basedir: {basedir} + remote: {self.source} >EOF r10k deploy environment -p From 010047cf7a4db633d589f7fe107ed284418a4890 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Igor=20Gali=C4=87?= Date: Wed, 6 Feb 2019 13:28:02 +0100 Subject: [PATCH 17/29] add --debug to puppet, libioc will filter it out anyway --- libioc/Provisioning/puppet.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libioc/Provisioning/puppet.py b/libioc/Provisioning/puppet.py index 9b6f1608..fd156998 100644 --- a/libioc/Provisioning/puppet.py +++ b/libioc/Provisioning/puppet.py @@ -137,12 +137,12 @@ def generate_postinstall(self) -> str: remote: {self.source} >EOF - r10k deploy environment -p + r10k deploy environment -pv """ postinstall += """ - puppet apply /usr/local/etc/puppet/environments/manifests/site.pp + puppet apply --debug {basedir}/manifests/site.pp """ return postinstall From 12f538a9429ced1514d8b497b99e1298602af74c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Gr=C3=B6nke?= Date: Wed, 6 Feb 2019 12:54:55 +0000 Subject: [PATCH 18/29] correct puppet provisioner url to source property --- libioc/Provisioning/puppet.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/libioc/Provisioning/puppet.py b/libioc/Provisioning/puppet.py index fd156998..2418373d 100644 --- a/libioc/Provisioning/puppet.py +++ b/libioc/Provisioning/puppet.py @@ -114,7 +114,6 @@ def source(self, value: typing.Union[ "Source must be urllib.parse.ParseResult or absolute path" ) - @property def pkgs(self) -> typing.List[str]: """Return list of packages required for this Provisioning method.""" @@ -180,7 +179,7 @@ def provision( try: yield jailProvisioningAssetDownloadEvent.begin() pluginDefinition = ControlRepoDefinition( - url=urllib.parse.urlparse(self.source).geturl(), + source=urllib.parse.urlparse(self.source).geturl(), logger=self.jail.logger ) yield jailProvisioningAssetDownloadEvent.end() @@ -198,7 +197,7 @@ def provision( if not os.path.isdir(plugin_dataset_name): # only clone if it doesn't already exist git.Repo.clone_from( - pluginDefinition.url, + pluginDefinition.source, plugin_dataset.mountpoint ) From 0b2267819d053e9526c8c09f72b6c31d79a13c37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Gr=C3=B6nke?= Date: Wed, 6 Feb 2019 12:56:35 +0000 Subject: [PATCH 19/29] use interpolation for used variables --- libioc/Provisioning/puppet.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libioc/Provisioning/puppet.py b/libioc/Provisioning/puppet.py index 2418373d..81f7e37d 100644 --- a/libioc/Provisioning/puppet.py +++ b/libioc/Provisioning/puppet.py @@ -128,7 +128,7 @@ def generate_postinstall(self) -> str: """ if self.remote: - postinstall += """cat > /usr/local/etc/r10k/r10k.yml /usr/local/etc/r10k/r10k.yml str: """ - postinstall += """ + postinstall += f""" puppet apply --debug {basedir}/manifests/site.pp """ return postinstall From 8c946e5261d0031d01f6c0ef6936170708338eb9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Gr=C3=B6nke?= Date: Thu, 7 Feb 2019 12:13:19 +0000 Subject: [PATCH 20/29] raise correct ValueError instead of TypeError in Types.AbsolutePath and Types.Path --- libioc/Types.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libioc/Types.py b/libioc/Types.py index 28001c16..a0d6afa2 100644 --- a/libioc/Types.py +++ b/libioc/Types.py @@ -43,7 +43,7 @@ def __init__( raise TypeError("Path must be a string") if len(self.blacklist.findall(sequence)) > 0: - raise TypeError(f"Illegal path: {sequence}") + raise ValueError(f"Illegal path: {sequence}") self = sequence # type: ignore @@ -59,7 +59,7 @@ def __init__( raise TypeError("AbsolutePath must be a string or Path") if str(sequence).startswith("/") is False: - raise TypeError( + raise ValueError( f"Expected AbsolutePath to begin with /, but got: {sequence}" ) From c6ec4e9eab3957d60e91da7f16a4379870e0cc7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Gr=C3=B6nke?= Date: Thu, 7 Feb 2019 12:16:06 +0000 Subject: [PATCH 21/29] move path validation methods from config to Resource --- libioc/Config/Jail/File/Fstab.py | 34 ++++++++++++++++---------- libioc/Config/Jail/File/__init__.py | 37 ++--------------------------- libioc/Config/Type/JSON.py | 2 +- libioc/Config/Type/UCL.py | 2 +- libioc/Resource.py | 22 +++++++++++++++++ 5 files changed, 48 insertions(+), 49 deletions(-) diff --git a/libioc/Config/Jail/File/Fstab.py b/libioc/Config/Jail/File/Fstab.py index cfddcf47..87de9153 100644 --- a/libioc/Config/Jail/File/Fstab.py +++ b/libioc/Config/Jail/File/Fstab.py @@ -147,10 +147,7 @@ def __hash__(self) -> int: return hash(None) -class Fstab( - libioc.Config.Jail.File.ResourceConfig, - collections.MutableSequence -): +class Fstab(collections.MutableSequence): """ Fstab configuration file wrapper. @@ -203,10 +200,7 @@ def path(self) -> str: path = self.file else: path = f"{self.jail.dataset.mountpoint}/{self.file}" - self._require_path_relative_to_resource( - filepath=path, - resource=self.jail - ) + self.jail._require_relative_path(path) return path @@ -400,7 +394,16 @@ def add_line( Use save() to write changes to the fstab file. """ - if self.__contains__(line): + if type(line) == FstabLine: + if self.jail._is_path_relative(line["destination"]) is False: + line = FstabLine(line) # clone to prevent mutation + line["destination"] = libioc.Types.AbsolutePath("/".join([ + self.jail.root_path, + line["destination"].lstrip("/") + ])) + + line_already_exists = self.__contains__(line) + if line_already_exists: destination = line["destination"] if replace is True: self.logger.verbose( @@ -425,11 +428,13 @@ def add_line( if type(line) == FstabLine: # destination is always relative to the jail resource - if line["destination"].startswith(self.jail.root_path) is False: - line["destination"] = libioc.Types.AbsolutePath("/".join([ + if self.jail._is_path_relative(line["destination"]) is False: + _destination = libioc.Types.AbsolutePath("/".join([ self.jail.root_path, line["destination"].strip("/") ])) + self.jail._require_relative_path(_destination) + line["destination"] = _destination libioc.helpers.require_no_symlink(str(line["destination"])) @@ -442,12 +447,17 @@ def add_line( os.makedirs(line["destination"], 0o700) if (auto_mount_jail and self.jail.running) is True: + destination = line["destination"] + self.jail._require_relative_path(destination) + self.logger.verbose( + f"auto-mount {destination}" + ) mount_command = [ "/sbin/mount", "-o", line["options"], "-t", line["type"], line["source"], - line["destination"] + destination ] libioc.helpers.exec(mount_command, logger=self.logger) _source = line["source"] diff --git a/libioc/Config/Jail/File/__init__.py b/libioc/Config/Jail/File/__init__.py index 974e0f91..362cdc63 100644 --- a/libioc/Config/Jail/File/__init__.py +++ b/libioc/Config/Jail/File/__init__.py @@ -30,36 +30,6 @@ import libioc.helpers_object import libioc.LaunchableResource - -class ResourceConfig: - """Shared abstract code between various config files in a resource.""" - - def _require_path_relative_to_resource( - self, - filepath: str, - resource: 'libioc.LaunchableResource.LaunchableResource' - ) -> None: - - if self._is_path_relative_to_resource(filepath, resource) is False: - raise libioc.errors.SecurityViolationConfigJailEscape( - file=filepath - ) - - def _is_path_relative_to_resource( - self, - filepath: str, - resource: 'libioc.LaunchableResource.LaunchableResource' - ) -> bool: - - real_resource_path = self._resolve_path(resource.dataset.mountpoint) - real_file_path = self._resolve_path(filepath) - - return real_file_path.startswith(real_resource_path) - - def _resolve_path(self, filepath: str) -> str: - return os.path.realpath(os.path.abspath(filepath)) - - class ConfigFile(dict): """Abstraction of UCL file based config files in Resources.""" @@ -221,7 +191,7 @@ def __getitem__( return None -class ResourceConfigFile(ConfigFile, ResourceConfig): +class ResourceConfigFile(ConfigFile): """Abstraction of UCL file based config files in Resources.""" def __init__( @@ -238,8 +208,5 @@ def __init__( def path(self) -> str: """Absolute path to the file.""" path = f"{self.resource.root_dataset.mountpoint}/{self.file}" - self._require_path_relative_to_resource( - filepath=path, - resource=self.resource - ) + self.resource._require_relative_path(path) return os.path.abspath(path) diff --git a/libioc/Config/Type/JSON.py b/libioc/Config/Type/JSON.py index 9a3c776b..71b765fa 100644 --- a/libioc/Config/Type/JSON.py +++ b/libioc/Config/Type/JSON.py @@ -53,6 +53,6 @@ class DatasetConfigJSON( libioc.Config.Dataset.DatasetConfig, ConfigJSON ): - """ResourceConfig in JSON format.""" + """ConfigFile in JSON format.""" pass diff --git a/libioc/Config/Type/UCL.py b/libioc/Config/Type/UCL.py index f4124eb7..1cad9e6a 100644 --- a/libioc/Config/Type/UCL.py +++ b/libioc/Config/Type/UCL.py @@ -52,6 +52,6 @@ class DatasetConfigUCL( libioc.Config.Dataset.DatasetConfig, ConfigUCL ): - """ResourceConfig in UCL format.""" + """ConfigFile in UCL format.""" pass diff --git a/libioc/Resource.py b/libioc/Resource.py index 96c12e23..521868b2 100644 --- a/libioc/Resource.py +++ b/libioc/Resource.py @@ -365,6 +365,28 @@ def save(self) -> None: "This needs to be implemented by the inheriting class" ) + def _require_relative_path( + self, + filepath: str, + ) -> None: + if self._is_path_relative(filepath) is False: + raise libioc.errors.SecurityViolationConfigJailEscape( + file=filepath + ) + + def _is_path_relative( + self, + filepath: str + ) -> bool: + + real_resource_path = self._resolve_path(self.dataset.mountpoint) + real_file_path = self._resolve_path(filepath) + + return real_file_path.startswith(real_resource_path) + + def _resolve_path(self, filepath: str) -> str: + return os.path.realpath(os.path.abspath(filepath)) + class DefaultResource(Resource): """The resource storing the default configuration.""" From b98a7d9f50fc5f4ba4f736e0d0ff0bc76dd44720 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Gr=C3=B6nke?= Date: Thu, 7 Feb 2019 12:16:29 +0000 Subject: [PATCH 22/29] ToDo: remove unused jailed argument --- libioc/Jail.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libioc/Jail.py b/libioc/Jail.py index b3063ba7..3b5651fc 100644 --- a/libioc/Jail.py +++ b/libioc/Jail.py @@ -726,7 +726,7 @@ def _wrap_hook_script_command( self, commands: typing.Optional[typing.Union[str, typing.List[str]]], ignore_errors: bool=True, - jailed: bool=False, + jailed: bool=False, # ToDo: remove unused argument write_env: bool=True ) -> typing.List[str]: From c671d88b3eb697b51f0d798b3e7d2269cdd159c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Gr=C3=B6nke?= Date: Thu, 7 Feb 2019 12:18:42 +0000 Subject: [PATCH 23/29] docstring on JailGenerator._ensure_script_dir --- libioc/Jail.py | 1 + 1 file changed, 1 insertion(+) diff --git a/libioc/Jail.py b/libioc/Jail.py index 3b5651fc..bf04b95b 100644 --- a/libioc/Jail.py +++ b/libioc/Jail.py @@ -862,6 +862,7 @@ def _run_hook(self, hook_name: str) -> typing.Optional[ raise NotImplementedError("_run_hook only supports start/stop") def _ensure_script_dir(self) -> None: + """Ensure that the launch scripts dir exists.""" realpath = os.path.realpath(self.launch_script_dir) if realpath.startswith(self.dataset.mountpoint) is False: raise libioc.errors.SecurityViolationConfigJailEscape( From c14904fb3ac37b1d60400cf923f8dcefa5994852 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Gr=C3=B6nke=20+=20Igor=20Gali=C4=87?= Date: Thu, 7 Feb 2019 12:20:53 +0000 Subject: [PATCH 24/29] refactor Provisioning --- libioc/Provisioning/__init__.py | 85 +++++++++++- libioc/Provisioning/puppet.py | 220 ++++++++++++++++++-------------- 2 files changed, 206 insertions(+), 99 deletions(-) diff --git a/libioc/Provisioning/__init__.py b/libioc/Provisioning/__init__.py index fb79ee15..346945e3 100644 --- a/libioc/Provisioning/__init__.py +++ b/libioc/Provisioning/__init__.py @@ -24,12 +24,91 @@ # POSSIBILITY OF SUCH DAMAGE. """iocage provisioning prototype.""" import typing +import urllib.parse import libioc.errors +import libioc.Types import libioc.helpers import libioc.Provisioning.ix import libioc.Provisioning.puppet +_SourceType = typing.Union[ + urllib.parse.ParseResult, + libioc.Types.AbsolutePath, +] +_SourceInputType = typing.Union[_SourceType, str] + +class Source(str): + + _value: _SourceType + + def __init__( + self, + value: _SourceInputType + ) -> None: + self.value = value + + @property + def value(self) -> _SourceType: + return self._value + + @value.setter + def value(self, value: _SourceInputType) -> None: + + if isinstance(value, libioc.Types.AbsolutePath) is True: + self._value = typing.Cast(libioc.Types.AbsolutePath, value) + return + elif isinstance(value, urllib.parse.ParseResult) is True: + url = typing.Cast(urllib.parse.ParseResult, value) + self.__require_valid_url(url) + self._value = _value + return + elif isinstance(value, str) is False: + raise TypeError( + f"Input must be URL, AbsolutePath or str, but was {type(value)}" + ) + + try: + self._value = libioc.Types.AbsolutePath(value) + return + except ValueError as e: + pass + + try: + url = urllib.parse.urlparse(value) + self.__require_valid_url(url) + self._value = url + return + except ValueError: + pass + + raise ValueError("Provisioning Source must be AbsolutePath or URL") + + @property + def local(self) -> bool: + """Return True when the source is local.""" + return (isinstance(self.value, libioc.Types.AbsolutePath) is True) + + @property + def remote(self) -> bool: + """Return True when the source is a remote URL.""" + return (self.local is False) + + def __require_valid_url(self, url: urllib.parse.ParseResult) -> None: + if url.scheme not in ("https", "http", "ssh", "git"): + raise ValueError(f"Invalid Source Scheme: {url.scheme}") + + def __str__(self) -> str: + """Return the Provisioning Source as string.""" + value = self.value + if isinstance(value, urllib.parse.ParseResult) is True: + return value.geturl() + else: + return str(value) + + def __repr__(self) -> str: + return f"" + class Prototype: @@ -47,14 +126,14 @@ def method(self) -> str: self.__METHOD @property - def source(self) -> typing.Optional[str]: + def source(self) -> typing.Optional[Source]: config_value = self.jail.config["provisioning.source"] - return None if (config_value is None) else str(config_value) + return None if (config_value is None) else Source(config_value) @property def rev(self) -> typing.Optional[str]: config_value = self.jail.config["provisioning.rev"] - return None if (config_value is None) else str(config_value) + return None if (config_value is None) else str(Source(config_value)) def check_requirements(self) -> None: """Check requirements before executing the provisioner.""" diff --git a/libioc/Provisioning/puppet.py b/libioc/Provisioning/puppet.py index 81f7e37d..1aa9c8d7 100644 --- a/libioc/Provisioning/puppet.py +++ b/libioc/Provisioning/puppet.py @@ -24,7 +24,6 @@ """ioc provisioner for use with `puppet apply`.""" import typing import os.path -import urllib.parse import git @@ -51,100 +50,55 @@ def __init__( ) +class R10kDeployEvent(libioc.events.JailEvent): + """Deploy control repo and install puppet modules.""" + + pass + + +class PuppetApplyEvent(libioc.events.JailEvent): + """Apply the puppet manifest.""" + + pass + + class ControlRepoDefinition(dict): """Puppet control-repo definition.""" - _source: str + __source: str _pkgs: typing.List[str] def __init__( self, - source: typing.Union[ - urllib.parse.DefragResult, - libioc.Types.AbsolutePath - ], + source: 'libioc.Provisioning.Source', logger: 'libioc.Logger.Logger' ) -> None: self.logger = logger - if isinstance(source, libioc.Types.AbsolutePath) is False \ - and isinstance(source, urllib.parse.ParseResult) is False: - raise TypeError("Source must be an URL or an absolute path") self.source = source - self._pkgs = ['puppet6'] # make this a Global Varialbe - if isinstance(source, libioc.Types.AbsolutePath) is False: + if source.remote is True: self._pkgs += 'rubygem-r10k' @property - def local(self) -> bool: - """Return whether this control repo resides locally.""" - _source = self.source - if isinstance(_source, libioc.Types.AbsolutePath) is False: - return False - return True - - @property - def remote(self) -> bool: - """Return whether this control repo resides locally.""" - return not self.local - - @property - def source(self) -> str: + def source( + self + ) -> 'libioc.Provisioning.Source': """Return the Puppet Control-Repo URL.""" - return self._source + return self.__source @source.setter - def source(self, value: typing.Union[ - urllib.parse.ParseResult, - libioc.Types.AbsolutePath - ]) -> None: + def source(self, source: 'libioc.Provisioning.Source') -> None: """Set the Puppet Control-Repo URL.""" - _source: urllib.parse.ParseResult - if isinstance(value, urllib.parse.ParseResult) is True: - _source = typing.cast(urllib.parse.ParseResult, value) - if _source.fragment != "": - raise ValueError("URL may not contain fragment") - - self._source = _source.geturl() - elif isinstance(value, libioc.Types.AbsolutePath) is True: - _source = value - else: - raise TypeError( - "Source must be urllib.parse.ParseResult or absolute path" - ) + if isinstance(source, libioc.Provisioning.Source) is False: + raise TypeError("Source must be libioc.Provisioning.Source") + self.__source = source @property def pkgs(self) -> typing.List[str]: """Return list of packages required for this Provisioning method.""" return self._pkgs - def generate_postinstall(self) -> str: - """Return list of strings representing our postinstall.""" - basedir = "/usr/local/etc/puppet/environments" - postinstall = """#!/bin/sh - set -eu - - """ - - if self.remote: - postinstall += f"""cat > /usr/local/etc/r10k/r10k.yml EOF - - r10k deploy environment -pv - - """ - - postinstall += f""" - puppet apply --debug {basedir}/manifests/site.pp - """ - return postinstall - def provision( self: 'libioc.Provisioning.Prototype', @@ -178,8 +132,14 @@ def provision( # download / mount provisioning assets try: yield jailProvisioningAssetDownloadEvent.begin() + if self.source is None: + raise libioc.errors.InvalidJailConfigValue( + property_name="provisioning.source", + reason="Source may not be empty", + logger=self.jail.logger + ) pluginDefinition = ControlRepoDefinition( - source=urllib.parse.urlparse(self.source).geturl(), + source=self.source, logger=self.jail.logger ) yield jailProvisioningAssetDownloadEvent.end() @@ -187,7 +147,7 @@ def provision( yield jailProvisioningAssetDownloadEvent.fail(e) raise e - if pluginDefinition.remote: + if self.source.remote: mode = 'rw' # we'll need to run r10k here.. plugin_dataset_name = f"{self.jail.dataset.name}/puppet" plugin_dataset = self.zfs.get_or_create_dataset( @@ -206,38 +166,106 @@ def provision( mode = 'ro' mount_source = self.source + if self.jail.stopped is True: + started = True + jailStartEvent = libioc.events.JailStart( + jail=self.jail, + scope=jailProvisioningEvent.scope + ) + yield jailStartEvent.begin() + yield from self.jail.start(event_scope=jailStartEvent.scope) + yield jailStartEvent.end() + else: + started = False + + pkg = libioc.Pkg.Pkg( + logger=self.jail.logger, + zfs=self.jail.zfs, + host=self.jail.host + ) + + yield from pkg.install( + jail=self.jail, + packages=list(pluginDefinition.pkgs), + event_scope=jailProvisioningEvent.scope + ) + + puppet_env_dir = "/usr/local/etc/puppet/environments" + + self.jail.logger.spam("Mounting puppet environment") self.jail.fstab.new_line( source=mount_source, - destination="/usr/local/etc/puppet", + destination=puppet_env_dir, options=mode, auto_create_destination=True, replace=True ) - self.jail.fstab.save() - - if "pkgs" in pluginDefinition.keys(): - pkg_packages = list(pluginDefinition.pkgs) - else: - pkg_packages = [] try: - pkg = libioc.Pkg.Pkg( - logger=self.jail.logger, - zfs=self.jail.zfs, - host=self.jail.host - ) - - postinstall_script = pluginDefinition.generate_postinstall() - postinstall = "{self.jail.abspath}/launch-scripts/provision.sh" - with open(postinstall, 'w') as f: - f.write(postinstall_script) + env = { + 'PATH': + '/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin' + } + if self.source.remote is True: + + r10kDeployEvent = R10kDeployEvent( + scope=jailProvisioningEvent.scope, + jail=self.jail + ) - yield from pkg.fetch_and_install( - jail=self.jail, - packages=pkg_packages, - postinstall=postinstall + yield r10kDeployEvent.begin() + try: + r10k_cfg = f"{self.jail.root_path}/usr/local/etc/r10k/r10k.yml" + self.jail.logger.verbose(f"Writing r10k config {r10k_cfg}") + with open(r10k_cfg, "w") as f: + f.write(f"""--- + :source: + puppet: + basedir: {puppet_env_dir} + remote: {self.source} + """) + + self.jail.logger.verbose("Deploying r10k config") + self.jail.exec([ + "r10k", + "deploy", + "environment", + "-pv" + ], env=env ) + except Exception as e: + yield r10kDeployEvent.fail(e) + raise e + yield r10kDeployEvent.end() + + puppetApplyEvent = PuppetApplyEvent( + scope=jailProvisioningEvent.scope, + jail=self.jail ) - except Exception as e: - yield jailProvisioningEvent.fail(e) - raise e - + yield puppetApplyEvent.begin() + try: + self.jail.exec([ + "puppet", + "apply", + "--debug", + "--logdest", + "syslog", + f"{puppet_env_dir}/production/manifests/site.pp" + ], env=env) + yield puppetApplyEvent.end() + except Exception as e: + yield puppetApplyEvent.fail(e) + raise e + + if started is True: + jailStopEvent = libioc.events.JailShutdown( + jail=self.jail, + scope=jailProvisioningEvent.scope + ) + yield jailStopEvent.begin() + yield from self.jail.stop(event_scope=jailStopEvent.scope) + yield jailStopEvent.end() + finally: + # in case anything fails the fstab mount needs to be removed + del self.jail.fstab[-1] + + yield jailProvisioningEvent.end() From 5adbb3b9b4007ba1a9fa8fcb5942b5beaad40d27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Gr=C3=B6nke=20+=20Igor=20Gali=C4=87?= Date: Thu, 7 Feb 2019 12:29:47 +0000 Subject: [PATCH 25/29] set /sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin:/root/bin globally in Jail.env --- libioc/Jail.py | 8 ++++++++ libioc/Provisioning/puppet.py | 8 ++------ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/libioc/Jail.py b/libioc/Jail.py index bf04b95b..21793715 100644 --- a/libioc/Jail.py +++ b/libioc/Jail.py @@ -2175,6 +2175,14 @@ def env(self) -> typing.Dict[str, str]: jail_env["IOC_JAIL_PATH"] = self.root_dataset.mountpoint jail_env["IOC_JID"] = str(self.jid) + jail_env["PATH"] = ":".join(( + "/sbin", + "/bin", + "/usr/sbin", + "/usr/bin", + "/usr/local/sbin", + "/usr/local/bin", + )) return jail_env diff --git a/libioc/Provisioning/puppet.py b/libioc/Provisioning/puppet.py index 1aa9c8d7..6db4b126 100644 --- a/libioc/Provisioning/puppet.py +++ b/libioc/Provisioning/puppet.py @@ -202,10 +202,6 @@ def provision( ) try: - env = { - 'PATH': - '/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin' - } if self.source.remote is True: r10kDeployEvent = R10kDeployEvent( @@ -231,7 +227,7 @@ def provision( "deploy", "environment", "-pv" - ], env=env ) + ]) except Exception as e: yield r10kDeployEvent.fail(e) raise e @@ -250,7 +246,7 @@ def provision( "--logdest", "syslog", f"{puppet_env_dir}/production/manifests/site.pp" - ], env=env) + ]) yield puppetApplyEvent.end() except Exception as e: yield puppetApplyEvent.fail(e) From d56ca127a16a3e328835b7d8906e3f781e0d3829 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Gr=C3=B6nke=20+=20Igor=20Gali=C4=87?= Date: Thu, 7 Feb 2019 13:01:31 +0000 Subject: [PATCH 26/29] fix typo in variable name in Fstab.insert --- libioc/Config/Jail/File/Fstab.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libioc/Config/Jail/File/Fstab.py b/libioc/Config/Jail/File/Fstab.py index 87de9153..d48fb1f1 100644 --- a/libioc/Config/Jail/File/Fstab.py +++ b/libioc/Config/Jail/File/Fstab.py @@ -642,7 +642,7 @@ def insert( # find FstabAutoPlaceholderLine instead line = list(filter( lambda x: isinstance(x, FstabAutoPlaceholderLine), - self._line + self._lines ))[0] real_index = self._lines.index(line) else: From 4d5ae503a6a2f55bd843230ec3ef4d6877775221 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Igor=20Gali=C4=87?= Date: Thu, 7 Feb 2019 21:38:54 +0100 Subject: [PATCH 27/29] puppet: fix for remote sources - do *not* git clone the control repo - instead, let r10k do the work - do do so, install git(-lite) Finally, fix r10k's config location (create dir), name (yml -> yaml), and content (indentation) --- libioc/Provisioning/puppet.py | 29 +++++++++++------------------ 1 file changed, 11 insertions(+), 18 deletions(-) diff --git a/libioc/Provisioning/puppet.py b/libioc/Provisioning/puppet.py index 6db4b126..4cad45d6 100644 --- a/libioc/Provisioning/puppet.py +++ b/libioc/Provisioning/puppet.py @@ -25,8 +25,6 @@ import typing import os.path -import git - import libioc.errors import libioc.events import libioc.Pkg @@ -78,7 +76,7 @@ def __init__( self.source = source self._pkgs = ['puppet6'] # make this a Global Varialbe if source.remote is True: - self._pkgs += 'rubygem-r10k' + self._pkgs += ['rubygem-r10k', 'git-lite'] @property def source( @@ -150,17 +148,10 @@ def provision( if self.source.remote: mode = 'rw' # we'll need to run r10k here.. plugin_dataset_name = f"{self.jail.dataset.name}/puppet" - plugin_dataset = self.zfs.get_or_create_dataset( + plugin_dataset = self.jail.zfs.get_or_create_dataset( plugin_dataset_name ) - if not os.path.isdir(plugin_dataset_name): - # only clone if it doesn't already exist - git.Repo.clone_from( - pluginDefinition.source, - plugin_dataset.mountpoint - ) - mount_source = plugin_dataset.mountpoint else: mode = 'ro' @@ -211,15 +202,17 @@ def provision( yield r10kDeployEvent.begin() try: - r10k_cfg = f"{self.jail.root_path}/usr/local/etc/r10k/r10k.yml" + r10k_dir = f"{self.jail.root_path}/usr/local/etc/r10k" + r10k_cfg = f"{r10k_dir}/r10k.yaml" + if not os.path.isdir(f"{r10k_dir}"): + os.mkdir(r10k_dir, mode=0o755) self.jail.logger.verbose(f"Writing r10k config {r10k_cfg}") with open(r10k_cfg, "w") as f: - f.write(f"""--- - :source: - puppet: - basedir: {puppet_env_dir} - remote: {self.source} - """) + f.write("\n".join([f"---", + ":sources:", + " puppet:", + f" basedir: {puppet_env_dir}", + f" remote: {self.source}\n"])) self.jail.logger.verbose("Deploying r10k config") self.jail.exec([ From ad979340fbac845febb02c7daf66832b9c6acb1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Igor=20Gali=C4=87?= Date: Thu, 7 Feb 2019 21:44:49 +0100 Subject: [PATCH 28/29] Better documentation! for provisioning.method=puppet --- libioc/Provisioning/puppet.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/libioc/Provisioning/puppet.py b/libioc/Provisioning/puppet.py index 4cad45d6..38132dc7 100644 --- a/libioc/Provisioning/puppet.py +++ b/libioc/Provisioning/puppet.py @@ -106,14 +106,20 @@ def provision( Provision the jail with Puppet apply using the supplied control-repo. The repo can either be a filesystem path, or a http[s]/git URL. - If the repo is a filesystem path, it will be mounted appropriately. - If the repo is a URL, it will be setup with `r10k`. + If the repo is a filesystem path, it will be mounted to + `/usr/local/etc/puppet/environments`. + If the repo is a URL, we will setup a ZFS dataset and mount that to + `/usr/local/etc/puppet/environments`, before deploying it with `r10k`. + + Example: ioc set \ provisioning.method=puppet \ - provisioning.source=http://example.com/my/puppet-env \ - myjail + provisioning.source=http://github.com/bsdci/puppet-control-repo \ + webserver + This should install a webserver that listens on port 80, and delivers a + Hello-World HTML site. """ events = libioc.events jailProvisioningEvent = events.JailProvisioning( From a0f8d2a5e937d58e7076c607de8f9f2057f7f68f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Igor=20Gali=C4=87?= Date: Thu, 7 Feb 2019 21:54:31 +0100 Subject: [PATCH 29/29] fix flake8 complaints --- libioc/Provisioning/puppet.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/libioc/Provisioning/puppet.py b/libioc/Provisioning/puppet.py index 38132dc7..660c9dce 100644 --- a/libioc/Provisioning/puppet.py +++ b/libioc/Provisioning/puppet.py @@ -215,10 +215,10 @@ def provision( self.jail.logger.verbose(f"Writing r10k config {r10k_cfg}") with open(r10k_cfg, "w") as f: f.write("\n".join([f"---", - ":sources:", - " puppet:", - f" basedir: {puppet_env_dir}", - f" remote: {self.source}\n"])) + ":sources:", + " puppet:", + f" basedir: {puppet_env_dir}", + f" remote: {self.source}\n"])) self.jail.logger.verbose("Deploying r10k config") self.jail.exec([