From 4fdbb3887a15b9c54d078845753884fa2ada8168 Mon Sep 17 00:00:00 2001 From: Trevor Benson Date: Mon, 25 Nov 2024 16:21:32 -0800 Subject: [PATCH] [collect] Node and Inherit config files Signed-off-by: Trevor Benson --- sos/collector/__init__.py | 10 +++++++ sos/collector/sosnode.py | 17 +++++++++++ sos/collector/transports/__init__.py | 31 +++++++++++++++++++++ sos/collector/transports/control_persist.py | 6 ++++ sos/collector/transports/juju.py | 7 +++++ sos/collector/transports/local.py | 3 ++ sos/collector/transports/oc.py | 7 +++++ sos/collector/transports/saltstack.py | 28 +++++++++++++++++-- 8 files changed, 107 insertions(+), 2 deletions(-) diff --git a/sos/collector/__init__.py b/sos/collector/__init__.py index d66b0ec547..8830dd2232 100644 --- a/sos/collector/__init__.py +++ b/sos/collector/__init__.py @@ -88,6 +88,7 @@ class SoSCollector(SoSComponent): 'encrypt_pass': '', 'group': None, 'image': '', + 'inherit_config_file': False, 'force_pull_image': True, 'skip_cleaning_files': [], 'jobs': 4, @@ -102,6 +103,7 @@ class SoSCollector(SoSComponent): 'map_file': '/etc/sos/cleaner/default_mapping', 'primary': '', 'namespaces': None, + 'node_config_file': None, 'nodes': [], 'no_env_vars': False, 'no_local': False, @@ -463,6 +465,14 @@ def add_parser_options(cls, parser): choices=['auto', 'https', 'ftp', 'sftp', 's3'], help="Manually specify the upload protocol") + collect_grp.add_argument('--inherit-config-file', default=False, + action='store_true', + help='Use the config file from the collector ' + 'for all nodes to use with sos report') + collect_grp.add_argument('--node-config-file', type=str, + default=None, + help='Path to an existing config file on the ' + 'nodes to use with sos report') # Group the cleaner options together cleaner_grp = parser.add_argument_group( diff --git a/sos/collector/sosnode.py b/sos/collector/sosnode.py index 9ead9705ab..f60ffed3d9 100644 --- a/sos/collector/sosnode.py +++ b/sos/collector/sosnode.py @@ -51,6 +51,8 @@ def __init__(self, address, commons, password=None, local_sudo=None, self.hostlen = commons['hostlen'] self.need_sudo = commons['need_sudo'] self.sos_options = commons['sos_options'] + self.node_config_file = self.opts.node_config_file + self.inherit_config_file = self.opts.inherit_config_file self.local = False self.host = None self.cluster = None @@ -762,6 +764,21 @@ def execute_sos_command(self): try: path = False checksum = False + config_file_arg = '' + if self.opts.node_config_file: + config_file_arg = f'--config-file={self.opts.node_config_file}' + elif self.opts.inherit_config_file: + if not self.local: + remote_config = f"/tmp/{self.tmpdir.split('/')[-1]}.conf" + self._transport.copy_file_to_remote( + self.opts.config_file, + remote_config) + config_file_arg = f'--config-file={remote_config}' + else: + config_file_arg = ( + f'--config-file={self.opts.config_file}') + if config_file_arg: + self.sos_cmd = f"{self.sos_cmd} {config_file_arg}" res = self.run_command(self.sos_cmd, timeout=self.opts.timeout, use_shell=True, diff --git a/sos/collector/transports/__init__.py b/sos/collector/transports/__init__.py index 5780719333..350cf2a5db 100644 --- a/sos/collector/transports/__init__.py +++ b/sos/collector/transports/__init__.py @@ -352,6 +352,37 @@ def _get_hostname(self): self.log_info(f"Hostname set to {self._hostname}") return self._hostname + def copy_file_to_remote(self, fname, dest): + """Copy a local file, fname, to dest on the remote node + + :param fname: The name of the file to copy + :type fname: ``str`` + + :param dest: Where to save the file to remotely + :type dest: ``str`` + + :returns: True if file was successfully copied to remote, or False + :rtype: ``bool`` + """ + attempts = 0 + try: + while attempts < 3: + attempts += 1 + ret = self._copy_file_to_remote(fname, dest) + if ret: + return True + self.log_info(f"File copy attempt {attempts} failed") + self.log_info("File copy failed after 3 attempts") + return False + except Exception as err: + self.log_error("Exception encountered during config copy attempt " + f"{attempts} for {fname}: {err}") + raise err + + def _copy_file_to_remote(self, fname, dest): + raise NotImplementedError( + f"Transport {self.name} does not support file copying") + def retrieve_file(self, fname, dest): """Copy a remote file, fname, to dest on the local node diff --git a/sos/collector/transports/control_persist.py b/sos/collector/transports/control_persist.py index 9a814831e1..34efe9b7a6 100644 --- a/sos/collector/transports/control_persist.py +++ b/sos/collector/transports/control_persist.py @@ -193,6 +193,12 @@ def remote_exec(self): f"{self.opts.ssh_user}@{self.address}") return self.ssh_cmd + def _copy_file_to_remote(self, fname, dest): + cmd = (f"/usr/bin/scp -oControlPath={self.control_path} " + f"{fname} {self.opts.ssh_user}@{self.address}:{dest}") + res = sos_get_command_output(cmd, timeout=10) + return res['status'] == 0 + def _retrieve_file(self, fname, dest): cmd = (f"/usr/bin/scp -oControlPath={self.control_path} " f"{self.opts.ssh_user}@{self.address}:{fname} {dest}") diff --git a/sos/collector/transports/juju.py b/sos/collector/transports/juju.py index f0572ec3a7..5d300d77c3 100644 --- a/sos/collector/transports/juju.py +++ b/sos/collector/transports/juju.py @@ -72,6 +72,13 @@ def remote_exec(self): option = f"{model_option} {target_option}" return f"juju ssh {option}" + def _copy_file_to_remote(self, fname, dest): + model, unit = self.address.split(":") + model_option = f"-m {model}" if model else "" + cmd = f"juju scp {model_option} -- {fname} {unit}:{dest}" + res = sos_get_command_output(cmd, timeout=15) + return res["status"] == 0 + def _retrieve_file(self, fname, dest): self._chmod(fname) # juju scp needs the archive to be world-readable model, unit = self.address.split(":") diff --git a/sos/collector/transports/local.py b/sos/collector/transports/local.py index f40a210588..75fb96e442 100644 --- a/sos/collector/transports/local.py +++ b/sos/collector/transports/local.py @@ -41,6 +41,9 @@ def _retrieve_file(self, fname, dest): def _format_cmd_for_exec(self, cmd): return cmd + def _copy_file_to_remote(self, fname, dest): + return True + def _read_file(self, fname): if os.path.exists(fname): with open(fname, 'r', encoding='utf-8') as rfile: diff --git a/sos/collector/transports/oc.py b/sos/collector/transports/oc.py index b38f81a9cb..e304944f42 100644 --- a/sos/collector/transports/oc.py +++ b/sos/collector/transports/oc.py @@ -232,6 +232,13 @@ def remote_exec(self): return (f"oc -n {self.project} exec --request-timeout=0 " f"{self.pod_name} -- /bin/bash -c") + def _copy_file_to_remote(self, fname, dest): + result = self.run_oc("cp --retries", stderr=True) + flags = '' if "unknown flag" in result["output"] else '--retries=5' + cmd = self.run_oc(f"cp {flags} {fname} {self.pod_name}:{dest}", + timeout=15) + return cmd['status'] == 0 + def _retrieve_file(self, fname, dest): # check if --retries flag is available for given version of oc result = self.run_oc("cp --retries", stderr=True) diff --git a/sos/collector/transports/saltstack.py b/sos/collector/transports/saltstack.py index 5e817baac1..42d7ba4ff5 100644 --- a/sos/collector/transports/saltstack.py +++ b/sos/collector/transports/saltstack.py @@ -45,6 +45,14 @@ def run_command(self, cmd, timeout=180, need_root=False, env=None, ret['output'] = self._convert_output_json(ret['output']) return ret + def _salt_copy_file(self, node, fname, dest): + """ + Execute cp.get_file on the remote host using SaltStack Master + """ + cmd = f"salt-cp {node} {fname} {dest}" + res = sos_get_command_output(cmd, timeout=15) + return res['status'] == 0 + def _salt_retrieve_file(self, node, fname, dest): """ Execute cp.push on the remote host using SaltStack Master @@ -119,12 +127,28 @@ def remote_exec(self): salt_args = "--out json --static --no-color" return f"salt {salt_args} {self.address} cmd.shell " + def _copy_file_to_remote(self, fname, dest): + """Copy a file to the remote host using SaltStack Master + + Parameters + fname The path to the file on the master + dest The path to the destination directory on the remote host + + Returns + True if the file was copied, else False + """ + return ( + self._salt_copy_file(self.address, fname, dest) + if self.connected + else False + ) + def _retrieve_file(self, fname, dest): """Retrieve a file from the remote host using saltstack Parameters - fname The path to the file on the remote host - dest The path to the destination directory on the master + fname The path to the file on the remote host + dest The path to the destination directory on the master Returns True if the file was retrieved, else False