From 7f59c821da30ed1daa43811a9a0002359585d372 Mon Sep 17 00:00:00 2001 From: Joachim Jablon Date: Wed, 26 Sep 2018 14:25:52 +0200 Subject: [PATCH 01/11] Rename vault.py in cli.py --- vault_cli/{vault.py => cli.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename vault_cli/{vault.py => cli.py} (100%) diff --git a/vault_cli/vault.py b/vault_cli/cli.py similarity index 100% rename from vault_cli/vault.py rename to vault_cli/cli.py From 40a419ec7b98894417e0d24b2c4d40dbdcf36eeb Mon Sep 17 00:00:00 2001 From: Joachim Jablon Date: Thu, 27 Sep 2018 17:05:24 +0200 Subject: [PATCH 02/11] Refactor the tool to be usable as a Python lib too --- vault_cli/__init__.py | 4 + vault_cli/{cli.py => __main__.py} | 124 ++++--------------- vault_cli/backend.py | 96 --------------- vault_cli/client.py | 198 ++++++++++++++++++++++++++++++ vault_cli/hvac.py | 10 +- vault_cli/requests.py | 19 +-- vault_cli/settings.py | 90 ++++++++++++++ 7 files changed, 329 insertions(+), 212 deletions(-) rename vault_cli/{cli.py => __main__.py} (55%) delete mode 100644 vault_cli/backend.py create mode 100644 vault_cli/client.py create mode 100644 vault_cli/settings.py diff --git a/vault_cli/__init__.py b/vault_cli/__init__.py index 4f6586b..2d1fd1b 100644 --- a/vault_cli/__init__.py +++ b/vault_cli/__init__.py @@ -15,3 +15,7 @@ See the License for the specific language governing permissions and limitations under the License. """ +from vault_cli.client import get_client + + +__all__ = ['get_client'] diff --git a/vault_cli/cli.py b/vault_cli/__main__.py similarity index 55% rename from vault_cli/cli.py rename to vault_cli/__main__.py index 83e9ac2..0ade10c 100644 --- a/vault_cli/cli.py +++ b/vault_cli/__main__.py @@ -17,17 +17,12 @@ limitations under the License. """ -import os - import click import yaml -# Ordered by increasing priority -CONFIG_FILES = [ - '/etc/vault.yml', - '~/.vault.yml', - './.vault.yml', -] +from vault_cli import client +from vault_cli import settings + CONTEXT_SETTINGS = {'help_option_names': ['-h', '--help']} @@ -35,106 +30,58 @@ @click.group(context_settings=CONTEXT_SETTINGS) @click.pass_context @click.option('--url', '-U', help='URL of the vault instance', - default='https://localhost:8200') -@click.option('--verify/--no-verify', default=True, + default=settings.DEFAULTS['url']) +@click.option('--verify/--no-verify', default=settings.DEFAULTS['verify'], help='Verify HTTPS certificate') @click.option('--certificate', '-c', type=click.File('rb'), - help='The certificate to connect to vault') + help='Certificate to connect to vault') @click.option('--token', '-t', help='The token to connect to Vault') @click.option('--token-file', '-T', type=click.File('rb'), help='File which contains the token to connect to Vault') @click.option('--username', '-u', - help='The username used for userpass authentication') + help='Username used for userpass authentication') @click.option('--password-file', '-w', type=click.File('rb'), help='Can read from stdin if "-" is used as parameter') @click.option('--base-path', '-b', help='Base path for requests') -@click.option('--backend', default='requests', +@click.option('--backend', default=settings.DEFAULTS['backend'], help='Name of the backend to use (requests, hvac)') def cli(ctx, **kwargs): """ Interact with a Vault. See subcommands for details. """ backend = kwargs.pop("backend") - if backend == "requests": - from vault_cli import requests as backend_module - elif backend == "hvac": - from vault_cli import hvac as backend_module - else: - raise ValueError("Wrong backend value {}".format(backend)) try: - ctx.obj = backend_module.VaultSession(**kwargs) + ctx.obj = client.get_client_from_kwargs( + backend=backend, **kwargs) except ValueError as exc: raise click.UsageError(str(exc)) -def read_config_file(file_path): - try: - with open(os.path.expanduser(file_path), "r") as f: - config = yaml.safe_load(f) - except IOError: - return {} - config.pop("config", None) - - # Because we're modifying the dict during iteration, we need to - # consolidate the keys into a list - for key in list(config): - config[key.replace("-", "_")] = config.pop(key) - - _open_file(config, "certificate") - _open_file(config, "password_file") - _open_file(config, "token_file") - - return config - - -def _open_file(config, key): - """ - Replace file name with open file at the given key - in the config dict - """ - try: - config[key] = open(os.path.expanduser(config[key]), "rb") - except KeyError: - pass - - @cli.command("list") @click.argument('path', required=False, default='') @click.pass_obj -def list_(session, path): +def list_(client_obj, path): """ List all the secrets at the given path. Folders are listed too. If no path is given, list the objects at the root. """ - result = session.list_secrets(path=path) + result = client_obj.list_secrets(path=path) click.echo(result) @cli.command(name='get-all') @click.argument('path', required=False, nargs=-1) @click.pass_obj -def get_all(session, path): +def get_all(client_obj, path): """ Return multiple secrets. Return a single yaml with all the secrets located at the given paths. Folders are recursively explored. Without a path, explores all the vault. """ - result = {} - - # Just renaming the variable - paths = path - - if not path: - paths = [""] - - for path in paths: - secrets = session.get_recursive_secrets(path=path) - result.update(nested_keys(path, secrets)) - - if "" in result: - result.update(result.pop("")) + paths = path or [""] + result = client_obj.get_all(paths) if result: click.echo(yaml.safe_dump( result, @@ -142,21 +89,6 @@ def get_all(session, path): explicit_start=True)) -def nested_keys(path, value): - """ - >>> nested_path('test', 'foo') - {'test': 'foo'} - - >>> nested_path('test/bla', 'foo') - {'test': {'bla': 'foo'}} - """ - try: - base, subpath = path.strip('/').split('/', 1) - except ValueError: - return {path: value} - return {base: nested_keys(subpath, value)} - - @cli.command() @click.pass_obj @click.option('--text', @@ -164,11 +96,11 @@ def nested_keys(path, value): help=("--text implies --without-key. Returns the value in " "plain text format instead of yaml.")) @click.argument('name') -def get(session, text, name): +def get(client_obj, text, name): """ Return a single secret value. """ - secret = session.get_secret(path=name) + secret = client_obj.get_secret(path=name) if text: click.echo(secret) return @@ -183,7 +115,7 @@ def get(session, text, name): @click.option('--yaml', 'format_yaml', is_flag=True) @click.argument('name') @click.argument('value', nargs=-1) -def set_(session, format_yaml, name, value): +def set_(client_obj, format_yaml, name, value): """ Set a single secret to the given value(s). """ @@ -193,35 +125,23 @@ def set_(session, format_yaml, name, value): if format_yaml: value = yaml.safe_load(value) - session.put_secret(path=name, value=value) + client_obj.put_secret(path=name, value=value) click.echo('Done') @cli.command() @click.pass_obj @click.argument('name') -def delete(session, name): +def delete(client_obj, name): """ Deletes a single secret. """ - session.delete_secret(path=name) + client_obj.delete_secret(path=name) click.echo('Done') -def build_config_from_files(): - config = {} - config_files = CONFIG_FILES - - for potential_file in config_files: - config.update(read_config_file(potential_file)) - - return config - - def main(): - config = build_config_from_files() - - return cli(default_map=config) + return cli(default_map=settings.CONFIG) if __name__ == '__main__': diff --git a/vault_cli/backend.py b/vault_cli/backend.py deleted file mode 100644 index 9740193..0000000 --- a/vault_cli/backend.py +++ /dev/null @@ -1,96 +0,0 @@ -""" -Copyright 2018 PeopleDoc -Written by Yann Lachiver - Joachim Jablon - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" -import json - - -class VaultAPIException(Exception): - - def __init__(self, status_code, body, *args, **kwargs): - super(VaultAPIException, self).__init__(*args, **kwargs) - self.status_code = status_code - try: - self.error = '\n'.join(json.loads(body)['errors']) - except Exception: - self.error = body - - def __str__(self): - return 'status={} error="{}"'.format(self.status_code, self.error) - - -class VaultSessionBase(): - def __init__(self, url, verify, base_path, - certificate=None, token=None, username=None, - password_file=None, token_file=None): - self.init_session(url=url, verify=verify) - - self.base_path = base_path.rstrip("/") + "/" - - if token_file: - token = token_file.read().decode("utf-8").strip() - - if token: - self.authenticate_token(token) - elif certificate: - self.authenticate_certificate( - certificate.read().decode("utf-8").strip()) - elif username: - if not password_file: - raise ValueError('Cannot use username without password file') - password = password_file.read().decode("utf-8").strip() - self.authenticate_userpass(username=username, password=password) - - else: - raise ValueError("No authentication method supplied") - - def get_recursive_secrets(self, path): - result = {} - for key in self.list_secrets(path=path): - key_url = '/'.join([path.rstrip('/'), key]) if path else key - - if key_url.endswith('/'): - result[key.rstrip('/')] = self.get_recursive_secrets(key_url) - continue - - secret = self.get_secret(path=key_url) - if secret: - result[key] = secret - return result - - def init_session(self, url, verify): - raise NotImplementedError - - def authenticate_token(self, token): - raise NotImplementedError - - def authenticate_certificate(certificate): - raise NotImplementedError - - def authenticate_userpass(self, username, password): - raise NotImplementedError - - def list_secrets(self, path): - raise NotImplementedError - - def get_secret(self, path): - raise NotImplementedError - - def delete_secret(self, path): - raise NotImplementedError - - def put_secret(self, path, value): - raise NotImplementedError diff --git a/vault_cli/client.py b/vault_cli/client.py new file mode 100644 index 0000000..c54ba18 --- /dev/null +++ b/vault_cli/client.py @@ -0,0 +1,198 @@ +""" +Copyright 2018 PeopleDoc +Written by Yann Lachiver + Joachim Jablon + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import json + +from vault_cli import settings + + +def get_client(**kwargs): + """ + Reads the kwargs and associate them with the + config files and default values to produce + a configured client object ready to do calls. + + All parameters are optional. + + Parameters + ---------- + + url : str + URL of the vault instance (default: https://localhost:8200) + verify : bool + Verify HTTPS certificate (default: True) + certificate : str + Path to the certificate to connect to vault + token : str + Token to connect to Vault + token_file : str + File which contains the token to connect to Vault + username : str + Username used for userpass authentication + password_file : str + Path to the file containing the password for userpass authentication + base_path : str + Base path for requests + backend : str or callable + Backend or name of the backend to use ('requests', 'hvac') + + Returns + ------- + An instance of the appropriate subclass of VaultClientBase + (or whatever was provided as "backend") + + Client instance exposes the following methods: + - list_secrets(path) + Returns the name of all elements at the given path. + Folder names end with "/" + - get_secret(path) + Returns the value for the secret at the given path + - delete_secret(path) + Deletes the secret at the given path + - put_secret(path, value) + Writes the secret at the given path + - get_all(paths=None) + Given an iterable of path, recursively returns all + the secrets + """ + options = settings.get_vault_options(**kwargs) + backend = options.pop("backend") + return settings.get_client_from_kwargs(backend=backend, **options) + + +def get_client_from_kwargs(backend, **kwargs): + """ + Initializes a client object from the given final + kwargs. + """ + if backend == "requests": + from vault_cli import requests + client_class = requests.RequestsVaultClient + elif backend == "hvac": + from vault_cli import hvac + client_class = hvac.HVACVaultClient + elif callable(backend): + client_class = backend + else: + raise ValueError("Wrong backend value {}".format(backend)) + + return client_class(**kwargs) + + +class VaultAPIException(Exception): + + def __init__(self, status_code, body, *args, **kwargs): + super(VaultAPIException, self).__init__(*args, **kwargs) + self.status_code = status_code + try: + self.error = '\n'.join(json.loads(body)['errors']) + except Exception: + self.error = body + + def __str__(self): + return 'status={} error="{}"'.format(self.status_code, self.error) + + +class VaultClientBase(): + def __init__(self, url, verify, base_path, + certificate=None, token=None, username=None, + password_file=None, token_file=None): + self._init_session(url=url, verify=verify) + + self.base_path = base_path.rstrip("/") + "/" + + if token_file: + token = token_file.decode("utf-8").strip() + + if token: + self._authenticate_token(token) + elif certificate: + self.authenticate_certificate( + certificate.decode("utf-8").strip()) + elif username: + if not password_file: + raise ValueError('Cannot use username without password file') + password = password_file.decode("utf-8").strip() + self.authenticate_userpass(username=username, password=password) + + else: + raise ValueError("No authentication method supplied") + + def _get_recursive_secrets(self, path): + result = {} + for key in self.list_secrets(path=path): + key_url = '/'.join([path.rstrip('/'), key]) if path else key + + if key_url.endswith('/'): + result[key.rstrip('/')] = self._get_recursive_secrets(key_url) + continue + + secret = self.get_secret(path=key_url) + if secret: + result[key] = secret + return result + + def get_all(self, paths): + result = {} + + for path in paths: + secrets = self._get_recursive_secrets(path=path) + result.update(nested_keys(path, secrets)) + + if "" in result: + result.update(result.pop("")) + + return result + + def _init_session(self, url, verify): + raise NotImplementedError + + def _authenticate_token(self, token): + raise NotImplementedError + + def _authenticate_certificate(certificate): + raise NotImplementedError + + def _authenticate_userpass(self, username, password): + raise NotImplementedError + + def list_secrets(self, path): + raise NotImplementedError + + def get_secret(self, path): + raise NotImplementedError + + def delete_secret(self, path): + raise NotImplementedError + + def put_secret(self, path, value): + raise NotImplementedError + + +def nested_keys(path, value): + """ + >>> nested_path('test', 'foo') + {'test': 'foo'} + + >>> nested_path('test/bla', 'foo') + {'test': {'bla': 'foo'}} + """ + try: + base, subpath = path.strip('/').split('/', 1) + except ValueError: + return {path: value} + return {base: nested_keys(subpath, value)} diff --git a/vault_cli/hvac.py b/vault_cli/hvac.py index 28217d7..68ba0c0 100644 --- a/vault_cli/hvac.py +++ b/vault_cli/hvac.py @@ -19,18 +19,18 @@ import hvac from vault_cli.backend import VaultAPIException -from vault_cli.backend import VaultSessionBase +from vault_cli.backend import VaultClientBase -class VaultSession(VaultSessionBase): +class HVACVaultClient(VaultClientBase): - def init_session(self, url, verify): + def _init_session(self, url, verify): self.client = hvac.Client(url=url, verify=verify) - def authenticate_token(self, token): + def _authenticate_token(self, token): self.client.token = token - def authenticate_userpass(self, username, password): + def _authenticate_userpass(self, username, password): self.client.auth_userpass(username, password) def list_secrets(self, path): diff --git a/vault_cli/requests.py b/vault_cli/requests.py index 33ec835..b248c75 100644 --- a/vault_cli/requests.py +++ b/vault_cli/requests.py @@ -15,6 +15,7 @@ See the License for the specific language governing permissions and limitations under the License. """ + from __future__ import absolute_import import requests @@ -27,7 +28,7 @@ from urlparse import urljoin from vault_cli.backend import VaultAPIException -from vault_cli.backend import VaultSessionBase +from vault_cli.backend import VaultClientBase class Session(requests.Session): @@ -47,14 +48,14 @@ def merge_environment_settings(self, url, proxies, stream, verify, url, proxies, stream, verify, *args, **kwargs) -class VaultSession(VaultSessionBase): +class RequestsVaultClient(VaultClientBase): - def init_session(self, url, verify): + def _init_session(self, url, verify): self.session = self.create_session(verify) self.url = urljoin(url, "v1/") - def full_url(self, path=None): + def _full_url(self, path=None): url = urljoin(self.url, self.base_path) if path: return urljoin(url, path) @@ -72,7 +73,7 @@ def create_session(verify): session.verify = verify return session - def authenticate_token(self, token): + def _authenticate_token(self, token): self.session.headers.update({'X-Vault-Token': token}) def authenticate_userpass(self, username, password): @@ -90,7 +91,7 @@ def authenticate_userpass(self, username, password): response.status_code) def get_secrets(self, path): - url = self.full_url(path) + url = self._full_url(path) response = self.session.get(url) self.handle_error(response) json_response = response.json() @@ -101,18 +102,18 @@ def get_secret(self, path): return data['value'] def list_secrets(self, path): - url = self.full_url(path).rstrip('/') + url = self._full_url(path).rstrip('/') response = self.session.get(url, params={'list': 'true'}) self.handle_error(response) json_response = response.json() return json_response['data']['keys'] def put_secret(self, path, value): - url = self.full_url(path) + url = self._full_url(path) response = self.session.put(url, json={'value': value}) self.handle_error(response, requests.codes.no_content) def delete_secret(self, path): - url = self.full_url(path) + url = self._full_url(path) response = self.session.delete(url) self.handle_error(response, requests.codes.no_content) diff --git a/vault_cli/settings.py b/vault_cli/settings.py new file mode 100644 index 0000000..8616c64 --- /dev/null +++ b/vault_cli/settings.py @@ -0,0 +1,90 @@ +""" +Copyright 2018 PeopleDoc +Written by Yann Lachiver + Joachim Jablon + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import os +import yaml + + +# Ordered by increasing priority +CONFIG_FILES = [ + '/etc/vault.yml', + '~/.vault.yml', + './.vault.yml', +] + +DEFAULTS = { + 'backend': 'requests', + 'base_path': None, + 'certificate': None, + 'password_file': None, + 'token': None, + 'token_file': None, + 'url': 'https://localhost:8200', + 'username': None, + 'verify': True, +} + + +def read_config_file(file_path): + try: + with open(os.path.expanduser(file_path), "r") as f: + config = yaml.safe_load(f) + except IOError: + return {} + config.pop("config", None) + + # Because we're modifying the dict during iteration, we need to + # consolidate the keys into a list + for key in list(config): + config[key.replace("-", "_")] = config.pop(key) + + config["certificate"] = _read_file(config.get("certificate")) + config["password_file"] = _read_file(config.get("password_file")) + config["token_file"] = _read_file(config.get("token_file")) + + return config + + +def _read_file(path): + """ + Replace file name with open file at the given key + in the config dict + """ + if path: + with open(os.path.expanduser(path), 'rb') as file_handler: + return file_handler.read() + + +def build_config_from_files(config_files): + config = DEFAULTS.copy() + + for potential_file in config_files: + config.update(read_config_file(potential_file)) + + return config + + +# Make sure our config files are not re-read +# everytime we create a backend object +CONFIG = build_config_from_files(CONFIG_FILES) + + +def get_vault_options(**kwargs): + values = CONFIG.copy() + # TODO: Env vars here + values.update(kwargs) From 4e38187d743feb597f2fe07d8b82e0d7748583bb Mon Sep 17 00:00:00 2001 From: Joachim Jablon Date: Thu, 27 Sep 2018 17:16:06 +0200 Subject: [PATCH 03/11] Have a __main__.py and still a cli.py --- setup.cfg | 2 +- vault_cli/__main__.py | 127 +------------------------------------ vault_cli/cli.py | 143 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 145 insertions(+), 127 deletions(-) create mode 100644 vault_cli/cli.py diff --git a/setup.cfg b/setup.cfg index 49676bb..23bee34 100644 --- a/setup.cfg +++ b/setup.cfg @@ -28,7 +28,7 @@ install_requires = [options.entry_points] console_scripts = - vault = vault_cli.vault:main + vault = vault_cli.cli:main [options.extras_require] hvac = diff --git a/vault_cli/__main__.py b/vault_cli/__main__.py index 0ade10c..a0d6505 100644 --- a/vault_cli/__main__.py +++ b/vault_cli/__main__.py @@ -17,132 +17,7 @@ limitations under the License. """ -import click -import yaml - -from vault_cli import client -from vault_cli import settings - - -CONTEXT_SETTINGS = {'help_option_names': ['-h', '--help']} - - -@click.group(context_settings=CONTEXT_SETTINGS) -@click.pass_context -@click.option('--url', '-U', help='URL of the vault instance', - default=settings.DEFAULTS['url']) -@click.option('--verify/--no-verify', default=settings.DEFAULTS['verify'], - help='Verify HTTPS certificate') -@click.option('--certificate', '-c', type=click.File('rb'), - help='Certificate to connect to vault') -@click.option('--token', '-t', help='The token to connect to Vault') -@click.option('--token-file', '-T', type=click.File('rb'), - help='File which contains the token to connect to Vault') -@click.option('--username', '-u', - help='Username used for userpass authentication') -@click.option('--password-file', '-w', type=click.File('rb'), - help='Can read from stdin if "-" is used as parameter') -@click.option('--base-path', '-b', help='Base path for requests') -@click.option('--backend', default=settings.DEFAULTS['backend'], - help='Name of the backend to use (requests, hvac)') -def cli(ctx, **kwargs): - """ - Interact with a Vault. See subcommands for details. - """ - backend = kwargs.pop("backend") - - try: - ctx.obj = client.get_client_from_kwargs( - backend=backend, **kwargs) - except ValueError as exc: - raise click.UsageError(str(exc)) - - -@cli.command("list") -@click.argument('path', required=False, default='') -@click.pass_obj -def list_(client_obj, path): - """ - List all the secrets at the given path. Folders are listed too. If no path - is given, list the objects at the root. - """ - result = client_obj.list_secrets(path=path) - click.echo(result) - - -@cli.command(name='get-all') -@click.argument('path', required=False, nargs=-1) -@click.pass_obj -def get_all(client_obj, path): - """ - Return multiple secrets. Return a single yaml with all the secrets located - at the given paths. Folders are recursively explored. Without a path, - explores all the vault. - """ - paths = path or [""] - - result = client_obj.get_all(paths) - if result: - click.echo(yaml.safe_dump( - result, - default_flow_style=False, - explicit_start=True)) - - -@cli.command() -@click.pass_obj -@click.option('--text', - is_flag=True, - help=("--text implies --without-key. Returns the value in " - "plain text format instead of yaml.")) -@click.argument('name') -def get(client_obj, text, name): - """ - Return a single secret value. - """ - secret = client_obj.get_secret(path=name) - if text: - click.echo(secret) - return - - click.echo(yaml.safe_dump(secret, - default_flow_style=False, - explicit_start=True)) - - -@cli.command("set") -@click.pass_obj -@click.option('--yaml', 'format_yaml', is_flag=True) -@click.argument('name') -@click.argument('value', nargs=-1) -def set_(client_obj, format_yaml, name, value): - """ - Set a single secret to the given value(s). - """ - if len(value) == 1: - value = value[0] - - if format_yaml: - value = yaml.safe_load(value) - - client_obj.put_secret(path=name, value=value) - click.echo('Done') - - -@cli.command() -@click.pass_obj -@click.argument('name') -def delete(client_obj, name): - """ - Deletes a single secret. - """ - client_obj.delete_secret(path=name) - click.echo('Done') - - -def main(): - return cli(default_map=settings.CONFIG) - +from vault_cli.cli import main if __name__ == '__main__': main() diff --git a/vault_cli/cli.py b/vault_cli/cli.py new file mode 100644 index 0000000..f185d2b --- /dev/null +++ b/vault_cli/cli.py @@ -0,0 +1,143 @@ +""" +Copyright 2018 PeopleDoc +Written by Yann Lachiver + Joachim Jablon + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import click +import yaml + +from vault_cli import client +from vault_cli import settings + + +CONTEXT_SETTINGS = {'help_option_names': ['-h', '--help']} + + +@click.group(context_settings=CONTEXT_SETTINGS) +@click.pass_context +@click.option('--url', '-U', help='URL of the vault instance', + default=settings.DEFAULTS['url']) +@click.option('--verify/--no-verify', default=settings.DEFAULTS['verify'], + help='Verify HTTPS certificate') +@click.option('--certificate', '-c', type=click.File('rb'), + help='Certificate to connect to vault') +@click.option('--token', '-t', help='The token to connect to Vault') +@click.option('--token-file', '-T', type=click.File('rb'), + help='File which contains the token to connect to Vault') +@click.option('--username', '-u', + help='Username used for userpass authentication') +@click.option('--password-file', '-w', type=click.File('rb'), + help='Can read from stdin if "-" is used as parameter') +@click.option('--base-path', '-b', help='Base path for requests') +@click.option('--backend', default=settings.DEFAULTS['backend'], + help='Name of the backend to use (requests, hvac)') +def cli(ctx, **kwargs): + """ + Interact with a Vault. See subcommands for details. + """ + backend = kwargs.pop("backend") + + try: + ctx.obj = client.get_client_from_kwargs( + backend=backend, **kwargs) + except ValueError as exc: + raise click.UsageError(str(exc)) + + +@cli.command("list") +@click.argument('path', required=False, default='') +@click.pass_obj +def list_(client_obj, path): + """ + List all the secrets at the given path. Folders are listed too. If no path + is given, list the objects at the root. + """ + result = client_obj.list_secrets(path=path) + click.echo(result) + + +@cli.command(name='get-all') +@click.argument('path', required=False, nargs=-1) +@click.pass_obj +def get_all(client_obj, path): + """ + Return multiple secrets. Return a single yaml with all the secrets located + at the given paths. Folders are recursively explored. Without a path, + explores all the vault. + """ + paths = path or [""] + + result = client_obj.get_all(paths) + if result: + click.echo(yaml.safe_dump( + result, + default_flow_style=False, + explicit_start=True)) + + +@cli.command() +@click.pass_obj +@click.option('--text', + is_flag=True, + help=("--text implies --without-key. Returns the value in " + "plain text format instead of yaml.")) +@click.argument('name') +def get(client_obj, text, name): + """ + Return a single secret value. + """ + secret = client_obj.get_secret(path=name) + if text: + click.echo(secret) + return + + click.echo(yaml.safe_dump(secret, + default_flow_style=False, + explicit_start=True)) + + +@cli.command("set") +@click.pass_obj +@click.option('--yaml', 'format_yaml', is_flag=True) +@click.argument('name') +@click.argument('value', nargs=-1) +def set_(client_obj, format_yaml, name, value): + """ + Set a single secret to the given value(s). + """ + if len(value) == 1: + value = value[0] + + if format_yaml: + value = yaml.safe_load(value) + + client_obj.put_secret(path=name, value=value) + click.echo('Done') + + +@cli.command() +@click.pass_obj +@click.argument('name') +def delete(client_obj, name): + """ + Deletes a single secret. + """ + client_obj.delete_secret(path=name) + click.echo('Done') + + +def main(): + return cli(default_map=settings.CONFIG) From a7450970fa9d5f99e8265ce2c3af5ebd9407d8a9 Mon Sep 17 00:00:00 2001 From: Joachim Jablon Date: Thu, 27 Sep 2018 22:22:04 +0200 Subject: [PATCH 04/11] Adding a 100% unit test coverage --- .gitignore | 2 + .travis.yml | 23 +++++ conftest.py | 11 ++ setup.cfg | 6 ++ tests/test_vault_api.py | 48 --------- tests/unit/test_cli.py | 147 ++++++++++++++++++++++++++ tests/unit/test_client.py | 205 +++++++++++++++++++++++++++++++++++++ tests/unit/test_clients.py | 111 ++++++++++++++++++++ tests/unit/test_main.py | 18 ++++ tox.ini | 12 +++ vault_cli/__main__.py | 10 +- vault_cli/cli.py | 14 +-- vault_cli/client.py | 40 +++++--- vault_cli/hvac.py | 8 +- vault_cli/requests.py | 24 ++--- vault_cli/settings.py | 21 ++-- 16 files changed, 599 insertions(+), 101 deletions(-) create mode 100644 .travis.yml create mode 100644 conftest.py delete mode 100644 tests/test_vault_api.py create mode 100644 tests/unit/test_cli.py create mode 100644 tests/unit/test_client.py create mode 100644 tests/unit/test_clients.py create mode 100644 tests/unit/test_main.py create mode 100644 tox.ini diff --git a/.gitignore b/.gitignore index 54dcc8c..40c5ffe 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,5 @@ __pycache__/ build dist +.coverage +htmlcov diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..b2d792b --- /dev/null +++ b/.travis.yml @@ -0,0 +1,23 @@ +# Config file for automatic testing at travis-ci.org +language: python + +services: + - docker + +matrix: + include: + + - python: 3.6 + env: TOX_ENV=py36-unit-tests COVERAGE_FLAG=unit + + - python: 2.7 + env: TOX_ENV=py27-unit-tests COVERAGE_FLAG=unit + +install: + - pip install tox codecov + +script: + - tox -e $TOX_ENV + +after_success: +- bash <(curl -s https://codecov.io/bash) -c -F $COVERAGE_FLAG diff --git a/conftest.py b/conftest.py new file mode 100644 index 0000000..7113bb3 --- /dev/null +++ b/conftest.py @@ -0,0 +1,11 @@ +import pytest + +from vault_cli import settings + + +@pytest.fixture +def config(): + old = settings.CONFIG + settings.CONFIG = {} + yield settings.CONFIG + settings.CONFIG = old diff --git a/setup.cfg b/setup.cfg index 23bee34..0d5bae9 100644 --- a/setup.cfg +++ b/setup.cfg @@ -40,7 +40,13 @@ dev = test = pytest pytest-mock + requests-mock + pytest-cov + pytest-click [bdist_wheel] universal = 1 + +[tool:pytest] +addopts = --cov-report term-missing --cov-branch --cov-report html --cov-report term --cov=vault_cli -vv diff --git a/tests/test_vault_api.py b/tests/test_vault_api.py deleted file mode 100644 index 5c285d9..0000000 --- a/tests/test_vault_api.py +++ /dev/null @@ -1,48 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright (C) 2018 PeopleDoc, created by Pierre-Louis Bonicoli - -import io -import pytest -from urllib3.response import HTTPResponse - -import requests -from vault_cli.vault_python_api import create_session, userpass_authentication - - -@pytest.fixture -def mock_request(request, mocker): - response = request.getfuncargvalue('testcase') - - def send(self, request, stream=False, timeout=None, verify=True, cert=None, - proxies=None): - data = response['data'] - resp = HTTPResponse(body=io.BytesIO(data), preload_content=False) - resp.status = response['status'] - resp.reason = response['reason'] - resp.headers = {} - return self.build_response(request, resp) - - mocker.patch('requests.adapters.HTTPAdapter.send', autospec=True, - side_effect=send) - - -TEST_CASE = [ - { - 'data': b'404 page not found', - 'reason': 'Not Found', - 'status': 404, - } -] - - -@pytest.mark.parametrize('testcase', TEST_CASE) -def test_wrong_url(mocker, mock_request, testcase): - """Check that an exception doesn't occur when URL provided by user is - wrong""" - - session = create_session(True) - with pytest.raises(ValueError) as excinfo: - userpass_authentication(session, 'https://localhost:8200/', 'user', 'pass') - assert requests.adapters.HTTPAdapter.send.call_count == 1 - assert "Wrong username or password (HTTP code: 404)" == str(excinfo.value) diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py new file mode 100644 index 0000000..13cb552 --- /dev/null +++ b/tests/unit/test_cli.py @@ -0,0 +1,147 @@ +import yaml +import pytest + +from vault_cli import cli +from vault_cli import client + + +class FakeClient(client.VaultClientBase): + def __init__(self, **kwargs): + self.init_kwargs = kwargs + print(kwargs) + + def get_secret(self, path): + return "bar" + + def list_secrets(self, path): + return ["foo", "baz"] + + def set_secret(self, path, value): + self.set = [path, value] + + def delete_secret(self, path): + self.deleted = path + + +@pytest.fixture +def backend(mocker): + backend = FakeClient() + mocker.patch("vault_cli.requests.RequestsVaultClient", + return_value=backend) + yield backend + + +def test_bad_backend(cli_runner, backend): + result = cli_runner.invoke(cli.cli, ["--backend", "bad", "list"]) + + assert result.exit_code != 0 + assert "Error: Wrong backend value bad" in result.output + + +def test_options(cli_runner, mocker): + func = mocker.patch("vault_cli.client.get_client_from_kwargs") + result = cli_runner.invoke(cli.cli, [ + "--backend", "requests", + "--base-path", "bla", + "--certificate", __file__, + "--password-file", __file__, + "--token", "tok", + "--token-file", __file__, + "--url", "https://foo", + "--username", "user", + "--verify", + "list" + ]) + + assert result.exit_code == 0 + _, kwargs = func.call_args + assert set(kwargs) == { + "backend", + "base_path", + "certificate", + "password_file", + "token", + "token_file", + "url", + "username", + "verify", + } + assert kwargs["base_path"] == "bla" + assert kwargs["certificate"].name == __file__ + assert kwargs["password_file"].name == __file__ + assert kwargs["token"] == "tok" + assert kwargs["token_file"].name == __file__ + assert kwargs["url"] == "https://foo" + assert kwargs["username"] == "user" + assert kwargs["verify"] is True + + +def test_list(cli_runner, backend): + result = cli_runner.invoke(cli.cli, ["list"]) + + assert result.output == "['foo', 'baz']\n" + assert result.exit_code == 0 + + +def test_get_text(cli_runner, backend): + + result = cli_runner.invoke(cli.cli, ["get", "a", "--text"]) + + assert result.output == "bar\n" + assert result.exit_code == 0 + + +def test_get_yaml(cli_runner, backend): + result = cli_runner.invoke(cli.cli, ["get", "a"]) + + assert yaml.safe_load(result.output) == "bar" + assert result.exit_code == 0 + + +def test_get_all(cli_runner, backend): + + result = cli_runner.invoke(cli.cli, ["get-all", "a"]) + + assert yaml.safe_load(result.output) == {'a': {'baz': 'bar', 'foo': 'bar'}} + assert result.exit_code == 0 + + +def test_set(cli_runner, backend): + + result = cli_runner.invoke(cli.cli, ["set", "a", "b"]) + + assert result.exit_code == 0 + assert backend.set == ["a", "b"] + + +def test_set_list(cli_runner, backend): + + result = cli_runner.invoke(cli.cli, ["set", "a", "b", "c"]) + + assert result.exit_code == 0 + assert backend.set == ["a", ["b", "c"]] + + +def test_set_yaml(cli_runner, backend): + + result = cli_runner.invoke(cli.cli, ["set", "--yaml", "a", '{"b": "c"}']) + + assert result.exit_code == 0 + assert backend.set == ["a", {"b": "c"}] + + +def test_delete(cli_runner, backend): + + result = cli_runner.invoke(cli.cli, ["delete", "a"]) + + assert result.exit_code == 0 + assert backend.deleted == "a" + + +def test_main(mocker, config): + mock_cli = mocker.patch("vault_cli.cli.cli") + config.update({"bla": "blu"}) + + cli.main() + + mock_cli.assert_called_with(default_map={"bla": "blu"}) diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py new file mode 100644 index 0000000..1a8defe --- /dev/null +++ b/tests/unit/test_client.py @@ -0,0 +1,205 @@ +import io + +import pytest + +from vault_cli import client + + +@pytest.mark.parametrize("backend, mock", [ + ("requests", "vault_cli.requests.RequestsVaultClient"), + ("hvac", "vault_cli.hvac.HVACVaultClient"), +]) +def test_get_client_from_kwargs(mocker, backend, mock): + c = mocker.patch(mock) + client.get_client_from_kwargs(backend, a=1) + + c.assert_called_with(a=1) + + +def test_get_client_from_kwargs_custom(mocker): + backend = mocker.MagicMock() + client.get_client_from_kwargs(backend, a=1) + + backend.assert_called_with(a=1) + + +def test_get_client_from_kwargs_bad(mocker): + with pytest.raises(ValueError): + client.get_client_from_kwargs("nope") + + +def test_get_client(mocker, config): + config.update({"url": "yay"}) + backend = mocker.Mock() + + c = client.get_client(backend=backend, yo=True) + + backend.assert_called_with(yo=True, url="yay") + assert backend.return_value == c + + +@pytest.mark.parametrize("error, expected", [ + ("oh no", '''status=404 error="oh no"'''), + ('''{"errors": ["damn", "gosh"]}''', '''status=404 error="damn\ngosh"'''), +]) +def test_vault_api_exception(error, expected): + exc_str = str(client.VaultAPIException(404, error)) + + assert exc_str == expected + + +@pytest.mark.parametrize("func, args", [ + ("_init_session", "url verify"), + ("_authenticate_token", "token"), + ("_authenticate_certificate", "certificate"), + ("_authenticate_userpass", "username password"), + ("list_secrets", "path"), + ("get_secret", "path"), + ("delete_secret", "path"), + ("set_secret", "path value"), +]) +def test_vault_client_base_not_implemented(func, args): + class TestVaultClient(client.VaultClientBase): + def __init__(self): + pass + c = TestVaultClient() + + with pytest.raises(NotImplementedError): + getattr(c, func)(**{name: None for name in args.split()}) + + +@pytest.mark.parametrize("path, value, expected", [ + ('test', 'foo', {'test': 'foo'}), + ('test/bla', 'foo', {'test': {'bla': 'foo'}}), +]) +def test_nested_keys(path, value, expected): + assert client.nested_keys(path, value) == expected + + +def test_vault_client_base_call_init_session(): + called_with = {} + + class TestVaultClient(client.VaultClientBase): + def _init_session(self, **kwargs): + called_with.update(kwargs) + + def _authenticate_token(self, *args, **kwargs): + pass + + TestVaultClient(verify=False, url="yay", token="go", + base_path=None, certificate=None, username=None, + password_file=None, token_file=None) + + assert called_with == {"verify": False, "url": "yay"} + + +@pytest.mark.parametrize("test_kwargs, expected", [ + ({"token": "yay"}, ["token", "yay"]), + ({"token_file": io.BytesIO(b"yay")}, ["token", "yay"]), + ( + {"username": "a", "password_file": io.BytesIO(b"b")}, + ["userpass", "a", "b"] + ), + ({"certificate": io.BytesIO(b"cert")}, ["certificate", "cert"]), +]) +def test_vault_client_base_authenticate(test_kwargs, expected): + auth_params = [] + + class TestVaultClient(client.VaultClientBase): + def _init_session(self, **kwargs): + pass + + def _authenticate_token(self, token): + auth_params.extend(["token", token]) + + def _authenticate_certificate(self, certificate): + auth_params.extend(["certificate", certificate]) + + def _authenticate_userpass(self, username, password): + auth_params.extend(["userpass", username, password]) + + kwargs = {"token": None, + "token_file": None, + "username": None, "password_file": None, + "certificate": None} + kwargs.update(test_kwargs) + TestVaultClient(verify=False, url=None, base_path=None, + **kwargs) + + assert auth_params == expected + + +def test_vault_client_base_username_without_password(): + + class TestVaultClient(client.VaultClientBase): + def _init_session(self, **kwargs): + pass + + with pytest.raises(ValueError): + TestVaultClient(username="yay", password_file=None, + verify=False, url="yay", token=None, + base_path=None, certificate=None, + token_file=None) + + +def test_vault_client_base_no_auth(): + + class TestVaultClient(client.VaultClientBase): + def _init_session(self, **kwargs): + pass + + with pytest.raises(ValueError): + TestVaultClient(username=None, password_file=None, + verify=False, url="yay", token=None, + base_path=None, certificate=None, + token_file=None) + + +def test_vault_client_base_get_recursive_secrets(): + + class TestVaultClient(client.VaultClientBase): + def __init__(self): + pass + + def list_secrets(self, path): + return { + "": ["a", "b/"], + "b": ["c"] + }[path] + + def get_secret(self, path): + return { + "a": "secret-a", + "b/c": "secret-bc", + }[path] + + result = TestVaultClient()._get_recursive_secrets("") + + assert result == {'a': 'secret-a', 'b': {'c': 'secret-bc'}} + + +def test_vault_client_base_get_all(): + + class TestVaultClient(client.VaultClientBase): + def __init__(self): + pass + + def list_secrets(self, path): + return { + "": ["a/", "b"], + "a": ["c"] + }[path] + + def get_secret(self, path): + return { + "a/c": "secret-ac", + "b": "secret-b", + }[path] + + result = TestVaultClient().get_all(["a", ""]) + + assert result == {'a': {'c': 'secret-ac'}, 'b': 'secret-b'} + + result = TestVaultClient().get_all(["a"]) + + assert result == {'a': {'c': 'secret-ac'}} diff --git a/tests/unit/test_clients.py b/tests/unit/test_clients.py new file mode 100644 index 0000000..1d3503d --- /dev/null +++ b/tests/unit/test_clients.py @@ -0,0 +1,111 @@ +import io + +import pytest + +from vault_cli import client + + +def get_client(backend, **additional_kwargs): + kwargs = { + "backend": backend, + "url": "http://vault:8000", + "verify": True, + "base_path": "bla", + "certificate": None, + "token": "tok", + "username": None, + "password_file": None, + "token_file": None, + } + kwargs.update(additional_kwargs) + return client.get_client_from_kwargs(**kwargs) + + +@pytest.mark.parametrize("backend", ["requests", "hvac"]) +def test_token(requests_mock, backend): + client_obj = get_client(backend) + requests_mock.get("http://vault:8000/v1/bla/a", + request_headers={'X-Vault-Token': 'tok'}, + json={"data": {"value": "b"}}) + + client_obj.get_secret("a") + + assert requests_mock.called + + +@pytest.mark.parametrize("backend", ["requests", "hvac"]) +def test_userpass(requests_mock, backend): + requests_mock.post("http://vault:8000/v1/auth/userpass/login/myuser", + json={"auth": {"client_token": "newtok"}}) + + # Initialize a client, check that we get a token + client_obj = get_client( + backend=backend, + token=None, + username="myuser", + password_file=io.BytesIO(b"pass"), + ) + + # Check that the token is used + requests_mock.get("http://vault:8000/v1/bla/a", + request_headers={'X-Vault-Token': 'newtok'}, + json={"data": {"value": "b"}}) + assert client_obj.get_secret("a") == "b" + + # Check that we sent the right pasword + assert requests_mock.request_history[0].json() == {'password': 'pass'} + + +@pytest.mark.parametrize("backend", ["requests", "hvac"]) +def test_get_secret(requests_mock, backend): + client_obj = get_client(backend) + requests_mock.get("http://vault:8000/v1/bla/a", + json={"data": {"value": "b"}}) + assert client_obj.get_secret("a") == "b" + + +@pytest.mark.parametrize("backend", ["requests", "hvac"]) +def test_get_secret_not_found(requests_mock, backend): + client_obj = get_client(backend) + requests_mock.get("http://vault:8000/v1/bla/a", status_code=404, + json={"errors": ["Not found"]}) + with pytest.raises(client.VaultAPIException): + assert client_obj.get_secret("a") + + +@pytest.mark.parametrize("backend", ["requests", "hvac"]) +def test_get_secret_no_verify(requests_mock, backend): + client_obj = get_client(backend, verify=False) + requests_mock.get("http://vault:8000/v1/bla/a", + json={"data": {"value": "b"}}) + assert client_obj.get_secret("a") == "b" + assert requests_mock.last_request.verify is False + + +@pytest.mark.parametrize("backend", ["requests", "hvac"]) +def test_list_secrets(requests_mock, backend): + client_obj = get_client(backend) + requests_mock.get("http://vault:8000/v1/bla/a?list=True", + json={"data": {"keys": ["b"]}}) + assert client_obj.list_secrets("a") == ["b"] + + +@pytest.mark.parametrize("backend", ["requests", "hvac"]) +def test_delete_secret(requests_mock, backend): + client_obj = get_client(backend) + requests_mock.delete("http://vault:8000/v1/bla/a", status_code=204) + client_obj.delete_secret("a") + + assert requests_mock.called + + +@pytest.mark.parametrize("backend", ["requests", "hvac"]) +def test_set_secret(requests_mock, backend): + client_obj = get_client(backend) + # Both post and put can be used + requests_mock.put("http://vault:8000/v1/bla/a", status_code=204, json={}) + requests_mock.post("http://vault:8000/v1/bla/a", status_code=204, json={}) + client_obj.set_secret("a", "b") + + assert requests_mock.called + assert requests_mock.request_history[0].json() == {'value': 'b'} diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py new file mode 100644 index 0000000..9ed3053 --- /dev/null +++ b/tests/unit/test_main.py @@ -0,0 +1,18 @@ +from vault_cli import __main__ +from vault_cli import cli + + +def test_main(): + assert __main__.main == cli.main + + +def test_entrypoint(mocker): + main = mocker.patch("vault_cli.__main__.main") + + __main__.entrypoint("bla") + + main.assert_not_called() + + __main__.entrypoint("__main__") + + main.assert_called() diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..d7dd585 --- /dev/null +++ b/tox.ini @@ -0,0 +1,12 @@ +[tox] +envlist = + {py27,py36}-{unit}-tests,check-lint + +[testenv] +usedevelop = True +extras = + test + hvac +commands = + pip freeze -l + unit-tests: pytest tests/unit diff --git a/vault_cli/__main__.py b/vault_cli/__main__.py index a0d6505..99eaf31 100644 --- a/vault_cli/__main__.py +++ b/vault_cli/__main__.py @@ -19,5 +19,11 @@ from vault_cli.cli import main -if __name__ == '__main__': - main() + +# Shenanigans for coverage +def entrypoint(name): + if name == '__main__': + main() + + +entrypoint(__name__) diff --git a/vault_cli/cli.py b/vault_cli/cli.py index f185d2b..d1fe497 100644 --- a/vault_cli/cli.py +++ b/vault_cli/cli.py @@ -81,11 +81,11 @@ def get_all(client_obj, path): paths = path or [""] result = client_obj.get_all(paths) - if result: - click.echo(yaml.safe_dump( - result, - default_flow_style=False, - explicit_start=True)) + + click.echo(yaml.safe_dump( + result, + default_flow_style=False, + explicit_start=True)) @cli.command() @@ -120,11 +120,13 @@ def set_(client_obj, format_yaml, name, value): """ if len(value) == 1: value = value[0] + else: + value = list(value) if format_yaml: value = yaml.safe_load(value) - client_obj.put_secret(path=name, value=value) + client_obj.set_secret(path=name, value=value) click.echo('Done') diff --git a/vault_cli/client.py b/vault_cli/client.py index c54ba18..4378779 100644 --- a/vault_cli/client.py +++ b/vault_cli/client.py @@ -63,7 +63,7 @@ def get_client(**kwargs): Returns the value for the secret at the given path - delete_secret(path) Deletes the secret at the given path - - put_secret(path, value) + - set_secret(path, value) Writes the secret at the given path - get_all(paths=None) Given an iterable of path, recursively returns all @@ -71,7 +71,7 @@ def get_client(**kwargs): """ options = settings.get_vault_options(**kwargs) backend = options.pop("backend") - return settings.get_client_from_kwargs(backend=backend, **options) + return get_client_from_kwargs(backend=backend, **options) def get_client_from_kwargs(backend, **kwargs): @@ -109,41 +109,47 @@ def __str__(self): class VaultClientBase(): def __init__(self, url, verify, base_path, - certificate=None, token=None, username=None, - password_file=None, token_file=None): + certificate, token, username, + password_file, token_file): + """ + All parameters are mandatory but may be None + """ self._init_session(url=url, verify=verify) - self.base_path = base_path.rstrip("/") + "/" + self.base_path = (base_path or "").rstrip("/") + "/" if token_file: - token = token_file.decode("utf-8").strip() + token = token_file.read().decode("utf-8").strip() if token: self._authenticate_token(token) elif certificate: - self.authenticate_certificate( - certificate.decode("utf-8").strip()) + self._authenticate_certificate( + certificate.read().decode("utf-8").strip()) elif username: if not password_file: raise ValueError('Cannot use username without password file') - password = password_file.decode("utf-8").strip() - self.authenticate_userpass(username=username, password=password) + password = password_file.read().decode("utf-8").strip() + self._authenticate_userpass(username=username, password=password) else: raise ValueError("No authentication method supplied") def _get_recursive_secrets(self, path): result = {} + path = path.rstrip('/') for key in self.list_secrets(path=path): - key_url = '/'.join([path.rstrip('/'), key]) if path else key + key_url = '/'.join([path, key]) if path else key - if key_url.endswith('/'): - result[key.rstrip('/')] = self._get_recursive_secrets(key_url) + folder = key_url.endswith('/') + key = key.rstrip('/') + if folder: + result[key] = self._get_recursive_secrets(key_url) continue secret = self.get_secret(path=key_url) - if secret: - result[key] = secret + result[key] = secret + return result def get_all(self, paths): @@ -164,7 +170,7 @@ def _init_session(self, url, verify): def _authenticate_token(self, token): raise NotImplementedError - def _authenticate_certificate(certificate): + def _authenticate_certificate(self, certificate): raise NotImplementedError def _authenticate_userpass(self, username, password): @@ -179,7 +185,7 @@ def get_secret(self, path): def delete_secret(self, path): raise NotImplementedError - def put_secret(self, path, value): + def set_secret(self, path, value): raise NotImplementedError diff --git a/vault_cli/hvac.py b/vault_cli/hvac.py index 68ba0c0..751cd18 100644 --- a/vault_cli/hvac.py +++ b/vault_cli/hvac.py @@ -18,8 +18,8 @@ import hvac -from vault_cli.backend import VaultAPIException -from vault_cli.backend import VaultClientBase +from vault_cli.client import VaultAPIException +from vault_cli.client import VaultClientBase class HVACVaultClient(VaultClientBase): @@ -40,10 +40,10 @@ def get_secret(self, path): secret = self.client.read(self.base_path + path) if not secret: raise VaultAPIException(404, "Not found") - return ["data"]["value"] + return secret["data"]["value"] def delete_secret(self, path): self.client.delete(self.base_path + path) - def put_secret(self, path, value): + def set_secret(self, path, value): self.client.write(self.base_path + path, value=value) diff --git a/vault_cli/requests.py b/vault_cli/requests.py index b248c75..09bfc71 100644 --- a/vault_cli/requests.py +++ b/vault_cli/requests.py @@ -27,8 +27,8 @@ # Python 2 from urlparse import urljoin -from vault_cli.backend import VaultAPIException -from vault_cli.backend import VaultClientBase +from vault_cli.client import VaultAPIException +from vault_cli.client import VaultClientBase class Session(requests.Session): @@ -55,11 +55,9 @@ def _init_session(self, url, verify): self.url = urljoin(url, "v1/") - def _full_url(self, path=None): + def _full_url(self, path): url = urljoin(self.url, self.base_path) - if path: - return urljoin(url, path) - return url + return urljoin(url, path) @staticmethod def handle_error(response, expected_code=requests.codes.ok): @@ -76,19 +74,15 @@ def create_session(verify): def _authenticate_token(self, token): self.session.headers.update({'X-Vault-Token': token}) - def authenticate_userpass(self, username, password): + def _authenticate_userpass(self, username, password): data = {"password": password} response = self.session.post(self.url + 'auth/userpass/login/' + username, json=data, headers={}) self.handle_error(response) - if response.status_code == requests.codes.ok: - json_response = response.json() - self.session.headers.update( - {'X-Vault-Token': json_response.get('auth').get('client_token')}) - else: - raise ValueError('Wrong username or password (HTTP code: %s)' % - response.status_code) + json_response = response.json() + self.session.headers.update( + {'X-Vault-Token': json_response.get('auth').get('client_token')}) def get_secrets(self, path): url = self._full_url(path) @@ -108,7 +102,7 @@ def list_secrets(self, path): json_response = response.json() return json_response['data']['keys'] - def put_secret(self, path, value): + def set_secret(self, path, value): url = self._full_url(path) response = self.session.put(url, json={'value': value}) self.handle_error(response, requests.codes.no_content) diff --git a/vault_cli/settings.py b/vault_cli/settings.py index 8616c64..4ee2cc8 100644 --- a/vault_cli/settings.py +++ b/vault_cli/settings.py @@ -43,27 +43,27 @@ def read_config_file(file_path): try: with open(os.path.expanduser(file_path), "r") as f: - config = yaml.safe_load(f) + return yaml.safe_load(f) except IOError: return {} - config.pop("config", None) + +def clean_config(config): # Because we're modifying the dict during iteration, we need to # consolidate the keys into a list for key in list(config): config[key.replace("-", "_")] = config.pop(key) - config["certificate"] = _read_file(config.get("certificate")) - config["password_file"] = _read_file(config.get("password_file")) - config["token_file"] = _read_file(config.get("token_file")) + config["certificate"] = read_file(config.get("certificate")) + config["password_file"] = read_file(config.get("password_file")) + config["token_file"] = read_file(config.get("token_file")) return config -def _read_file(path): +def read_file(path): """ - Replace file name with open file at the given key - in the config dict + Returns the content of the pointed file """ if path: with open(os.path.expanduser(path), 'rb') as file_handler: @@ -74,7 +74,8 @@ def build_config_from_files(config_files): config = DEFAULTS.copy() for potential_file in config_files: - config.update(read_config_file(potential_file)) + partial = clean_config(read_config_file(potential_file)) + config.update(partial) return config @@ -88,3 +89,5 @@ def get_vault_options(**kwargs): values = CONFIG.copy() # TODO: Env vars here values.update(kwargs) + + return values From 8cb01cb0b98eb18c4b70537f0284959dff0f6bb3 Mon Sep 17 00:00:00 2001 From: Joachim Jablon Date: Thu, 27 Sep 2018 22:22:23 +0200 Subject: [PATCH 05/11] Adding integration test --- .travis.yml | 9 ++++ tests/integration/test_integration.py | 65 +++++++++++++++++++++++++++ tox.ini | 3 +- vault_cli/cli.py | 1 - vault_cli/client.py | 3 ++ vault_cli/settings.py | 5 ++- 6 files changed, 83 insertions(+), 3 deletions(-) create mode 100644 tests/integration/test_integration.py diff --git a/.travis.yml b/.travis.yml index b2d792b..663a907 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,9 +10,18 @@ matrix: - python: 3.6 env: TOX_ENV=py36-unit-tests COVERAGE_FLAG=unit + - python: 3.6 + env: TOX_ENV=py36-integration-tests COVERAGE_FLAG=integration + - python: 2.7 env: TOX_ENV=py27-unit-tests COVERAGE_FLAG=unit + - python: 2.7 + env: TOX_ENV=py27-integration-tests COVERAGE_FLAG=integration + +before_install: + - "[$COVERAGE_FLAG = integration] && ./dev-env" + install: - pip install tox codecov diff --git a/tests/integration/test_integration.py b/tests/integration/test_integration.py new file mode 100644 index 0000000..16413a0 --- /dev/null +++ b/tests/integration/test_integration.py @@ -0,0 +1,65 @@ +from vault_cli.cli import cli +from vault_cli import get_client +from vault_cli import settings + + +def call(cli_runner, *args): + call = cli_runner.invoke(cli, *args, default_map=settings.CONFIG) + assert call.exit_code == 0, call.output + return call + + +def test_integration_cli(cli_runner): + call(cli_runner, ["set", "a", "b"]) + + assert call(cli_runner, ["get", "a", "--text"]).output == "b\n" + + assert call(cli_runner, ["list"]).output == '''['a']\n''' + + call(cli_runner, ["set", "c/d", "e"]) + + assert call(cli_runner, ["get", "c/d"]).output == "--- e\n...\n\n" + + assert call(cli_runner, ["list"]).output == '''['a', 'c/']\n''' + + assert call(cli_runner, ["list", "c"]).output == '''['d']\n''' + + assert call(cli_runner, ["get-all", ""]).output == ("""--- +a: b +c: + d: e + +""") + + call(cli_runner, ["delete", "a"]) + + assert call(cli_runner, ["list"]).output == '''['c/']\n''' + + call(cli_runner, ["delete", "c/d"]) + + +def test_integration_lib(): + + client = get_client() + + client.set_secret("a", "b") + + assert client.get_secret("a") == "b" + + assert client.list_secrets("") == ["a"] + + client.set_secret("c/d", "e") + + assert client.get_secret("c/d") == "e" + + assert client.list_secrets("") == ["a", "c/"] + + assert client.list_secrets("c") == ["d"] + + assert client.get_all([""]) == {"a": "b", "c": {"d": "e"}} + + client.delete_secret("a") + + assert client.list_secrets("") == ["c/"] + + client.delete_secret("c/d") diff --git a/tox.ini b/tox.ini index d7dd585..820ea12 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] envlist = - {py27,py36}-{unit}-tests,check-lint + {py27,py36}-{integration,unit}-tests,check-lint [testenv] usedevelop = True @@ -10,3 +10,4 @@ extras = commands = pip freeze -l unit-tests: pytest tests/unit + integration-tests: pytest tests/integration diff --git a/vault_cli/cli.py b/vault_cli/cli.py index d1fe497..4adaa7a 100644 --- a/vault_cli/cli.py +++ b/vault_cli/cli.py @@ -49,7 +49,6 @@ def cli(ctx, **kwargs): Interact with a Vault. See subcommands for details. """ backend = kwargs.pop("backend") - try: ctx.obj = client.get_client_from_kwargs( backend=backend, **kwargs) diff --git a/vault_cli/client.py b/vault_cli/client.py index 4378779..dc64fe1 100644 --- a/vault_cli/client.py +++ b/vault_cli/client.py @@ -119,16 +119,19 @@ def __init__(self, url, verify, base_path, self.base_path = (base_path or "").rstrip("/") + "/" if token_file: + token_file.seek(0) token = token_file.read().decode("utf-8").strip() if token: self._authenticate_token(token) elif certificate: + certificate.seek(0) self._authenticate_certificate( certificate.read().decode("utf-8").strip()) elif username: if not password_file: raise ValueError('Cannot use username without password file') + password_file.seek(0) password = password_file.read().decode("utf-8").strip() self._authenticate_userpass(username=username, password=password) diff --git a/vault_cli/settings.py b/vault_cli/settings.py index 4ee2cc8..fd8df47 100644 --- a/vault_cli/settings.py +++ b/vault_cli/settings.py @@ -16,6 +16,7 @@ limitations under the License. """ +import io import os import yaml @@ -67,7 +68,9 @@ def read_file(path): """ if path: with open(os.path.expanduser(path), 'rb') as file_handler: - return file_handler.read() + file = io.BytesIO(file_handler.read()) + file.seek(0) + return file def build_config_from_files(config_files): From b6f20dd2b630528f5075d1d5ff32e5892e09811a Mon Sep 17 00:00:00 2001 From: Joachim Jablon Date: Fri, 28 Sep 2018 11:47:33 +0200 Subject: [PATCH 06/11] Rename password_file as password for client --- tests/unit/test_client.py | 10 +++++----- tests/unit/test_clients.py | 4 ++-- vault_cli/client.py | 14 +++++--------- vault_cli/settings.py | 4 ++-- 4 files changed, 14 insertions(+), 18 deletions(-) diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py index 1a8defe..d4853cf 100644 --- a/tests/unit/test_client.py +++ b/tests/unit/test_client.py @@ -88,7 +88,7 @@ def _authenticate_token(self, *args, **kwargs): TestVaultClient(verify=False, url="yay", token="go", base_path=None, certificate=None, username=None, - password_file=None, token_file=None) + password=None, token_file=None) assert called_with == {"verify": False, "url": "yay"} @@ -97,7 +97,7 @@ def _authenticate_token(self, *args, **kwargs): ({"token": "yay"}, ["token", "yay"]), ({"token_file": io.BytesIO(b"yay")}, ["token", "yay"]), ( - {"username": "a", "password_file": io.BytesIO(b"b")}, + {"username": "a", "password": io.BytesIO(b"b")}, ["userpass", "a", "b"] ), ({"certificate": io.BytesIO(b"cert")}, ["certificate", "cert"]), @@ -120,7 +120,7 @@ def _authenticate_userpass(self, username, password): kwargs = {"token": None, "token_file": None, - "username": None, "password_file": None, + "username": None, "password": None, "certificate": None} kwargs.update(test_kwargs) TestVaultClient(verify=False, url=None, base_path=None, @@ -136,7 +136,7 @@ def _init_session(self, **kwargs): pass with pytest.raises(ValueError): - TestVaultClient(username="yay", password_file=None, + TestVaultClient(username="yay", password=None, verify=False, url="yay", token=None, base_path=None, certificate=None, token_file=None) @@ -149,7 +149,7 @@ def _init_session(self, **kwargs): pass with pytest.raises(ValueError): - TestVaultClient(username=None, password_file=None, + TestVaultClient(username=None, password=None, verify=False, url="yay", token=None, base_path=None, certificate=None, token_file=None) diff --git a/tests/unit/test_clients.py b/tests/unit/test_clients.py index 1d3503d..5609cdb 100644 --- a/tests/unit/test_clients.py +++ b/tests/unit/test_clients.py @@ -14,7 +14,7 @@ def get_client(backend, **additional_kwargs): "certificate": None, "token": "tok", "username": None, - "password_file": None, + "password": None, "token_file": None, } kwargs.update(additional_kwargs) @@ -43,7 +43,7 @@ def test_userpass(requests_mock, backend): backend=backend, token=None, username="myuser", - password_file=io.BytesIO(b"pass"), + password=io.BytesIO(b"pass"), ) # Check that the token is used diff --git a/vault_cli/client.py b/vault_cli/client.py index dc64fe1..95e9e86 100644 --- a/vault_cli/client.py +++ b/vault_cli/client.py @@ -43,7 +43,7 @@ def get_client(**kwargs): File which contains the token to connect to Vault username : str Username used for userpass authentication - password_file : str + password : str Path to the file containing the password for userpass authentication base_path : str Base path for requests @@ -110,7 +110,7 @@ def __str__(self): class VaultClientBase(): def __init__(self, url, verify, base_path, certificate, token, username, - password_file, token_file): + password, token_file): """ All parameters are mandatory but may be None """ @@ -118,10 +118,6 @@ def __init__(self, url, verify, base_path, self.base_path = (base_path or "").rstrip("/") + "/" - if token_file: - token_file.seek(0) - token = token_file.read().decode("utf-8").strip() - if token: self._authenticate_token(token) elif certificate: @@ -129,10 +125,10 @@ def __init__(self, url, verify, base_path, self._authenticate_certificate( certificate.read().decode("utf-8").strip()) elif username: - if not password_file: + if not password: raise ValueError('Cannot use username without password file') - password_file.seek(0) - password = password_file.read().decode("utf-8").strip() + password.seek(0) + password = password.read().decode("utf-8").strip() self._authenticate_userpass(username=username, password=password) else: diff --git a/vault_cli/settings.py b/vault_cli/settings.py index fd8df47..af7c648 100644 --- a/vault_cli/settings.py +++ b/vault_cli/settings.py @@ -32,7 +32,7 @@ 'backend': 'requests', 'base_path': None, 'certificate': None, - 'password_file': None, + 'password': None, 'token': None, 'token_file': None, 'url': 'https://localhost:8200', @@ -56,7 +56,7 @@ def clean_config(config): config[key.replace("-", "_")] = config.pop(key) config["certificate"] = read_file(config.get("certificate")) - config["password_file"] = read_file(config.get("password_file")) + config["password"] = read_file(config.get("password")) config["token_file"] = read_file(config.get("token_file")) return config From edc4373defa10a8618f93f57f08572c5fddea1b8 Mon Sep 17 00:00:00 2001 From: Joachim Jablon Date: Fri, 28 Sep 2018 11:50:35 +0200 Subject: [PATCH 07/11] No more token_file in the client --- tests/unit/test_client.py | 10 +++------- tests/unit/test_clients.py | 1 - vault_cli/client.py | 10 ++-------- vault_cli/settings.py | 7 ++----- 4 files changed, 7 insertions(+), 21 deletions(-) diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py index d4853cf..57c614b 100644 --- a/tests/unit/test_client.py +++ b/tests/unit/test_client.py @@ -88,14 +88,13 @@ def _authenticate_token(self, *args, **kwargs): TestVaultClient(verify=False, url="yay", token="go", base_path=None, certificate=None, username=None, - password=None, token_file=None) + password=None) assert called_with == {"verify": False, "url": "yay"} @pytest.mark.parametrize("test_kwargs, expected", [ ({"token": "yay"}, ["token", "yay"]), - ({"token_file": io.BytesIO(b"yay")}, ["token", "yay"]), ( {"username": "a", "password": io.BytesIO(b"b")}, ["userpass", "a", "b"] @@ -119,7 +118,6 @@ def _authenticate_userpass(self, username, password): auth_params.extend(["userpass", username, password]) kwargs = {"token": None, - "token_file": None, "username": None, "password": None, "certificate": None} kwargs.update(test_kwargs) @@ -138,8 +136,7 @@ def _init_session(self, **kwargs): with pytest.raises(ValueError): TestVaultClient(username="yay", password=None, verify=False, url="yay", token=None, - base_path=None, certificate=None, - token_file=None) + base_path=None, certificate=None) def test_vault_client_base_no_auth(): @@ -151,8 +148,7 @@ def _init_session(self, **kwargs): with pytest.raises(ValueError): TestVaultClient(username=None, password=None, verify=False, url="yay", token=None, - base_path=None, certificate=None, - token_file=None) + base_path=None, certificate=None) def test_vault_client_base_get_recursive_secrets(): diff --git a/tests/unit/test_clients.py b/tests/unit/test_clients.py index 5609cdb..ac86cac 100644 --- a/tests/unit/test_clients.py +++ b/tests/unit/test_clients.py @@ -15,7 +15,6 @@ def get_client(backend, **additional_kwargs): "token": "tok", "username": None, "password": None, - "token_file": None, } kwargs.update(additional_kwargs) return client.get_client_from_kwargs(**kwargs) diff --git a/vault_cli/client.py b/vault_cli/client.py index 95e9e86..0dffd6c 100644 --- a/vault_cli/client.py +++ b/vault_cli/client.py @@ -39,8 +39,6 @@ def get_client(**kwargs): Path to the certificate to connect to vault token : str Token to connect to Vault - token_file : str - File which contains the token to connect to Vault username : str Username used for userpass authentication password : str @@ -110,7 +108,7 @@ def __str__(self): class VaultClientBase(): def __init__(self, url, verify, base_path, certificate, token, username, - password, token_file): + password): """ All parameters are mandatory but may be None """ @@ -121,14 +119,10 @@ def __init__(self, url, verify, base_path, if token: self._authenticate_token(token) elif certificate: - certificate.seek(0) - self._authenticate_certificate( - certificate.read().decode("utf-8").strip()) + self._authenticate_certificate(certificate) elif username: if not password: raise ValueError('Cannot use username without password file') - password.seek(0) - password = password.read().decode("utf-8").strip() self._authenticate_userpass(username=username, password=password) else: diff --git a/vault_cli/settings.py b/vault_cli/settings.py index af7c648..76078e5 100644 --- a/vault_cli/settings.py +++ b/vault_cli/settings.py @@ -34,7 +34,6 @@ 'certificate': None, 'password': None, 'token': None, - 'token_file': None, 'url': 'https://localhost:8200', 'username': None, 'verify': True, @@ -57,7 +56,7 @@ def clean_config(config): config["certificate"] = read_file(config.get("certificate")) config["password"] = read_file(config.get("password")) - config["token_file"] = read_file(config.get("token_file")) + config["token"] = read_file(config.get("token_file")) return config @@ -68,9 +67,7 @@ def read_file(path): """ if path: with open(os.path.expanduser(path), 'rb') as file_handler: - file = io.BytesIO(file_handler.read()) - file.seek(0) - return file + return file_handler.read().decode("utf-8").strip() def build_config_from_files(config_files): From 857a224573dba8c1f27e0d85238068a6f73ed444 Mon Sep 17 00:00:00 2001 From: Joachim Jablon Date: Fri, 28 Sep 2018 13:52:50 +0200 Subject: [PATCH 08/11] Improve file handling --- tests/unit/test_cli.py | 21 +++++----- tests/unit/test_client.py | 6 +-- tests/unit/test_clients.py | 4 +- tests/unit/test_settings.py | 76 +++++++++++++++++++++++++++++++++++++ vault_cli/cli.py | 25 +++++++----- vault_cli/settings.py | 56 ++++++++++++++++++--------- 6 files changed, 144 insertions(+), 44 deletions(-) create mode 100644 tests/unit/test_settings.py diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index 13cb552..c644fef 100644 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -40,37 +40,36 @@ def test_bad_backend(cli_runner, backend): def test_options(cli_runner, mocker): func = mocker.patch("vault_cli.client.get_client_from_kwargs") + mocker.patch("vault_cli.settings.read_file", + side_effect=lambda x: "content of {}".format(x)) result = cli_runner.invoke(cli.cli, [ "--backend", "requests", "--base-path", "bla", - "--certificate", __file__, - "--password-file", __file__, - "--token", "tok", - "--token-file", __file__, + "--certificate-file", "a", + "--password-file", "b", + "--token-file", "c", "--url", "https://foo", "--username", "user", "--verify", "list" ]) - assert result.exit_code == 0 + assert result.exit_code == 0, result.output _, kwargs = func.call_args assert set(kwargs) == { "backend", "base_path", "certificate", - "password_file", + "password", "token", - "token_file", "url", "username", "verify", } assert kwargs["base_path"] == "bla" - assert kwargs["certificate"].name == __file__ - assert kwargs["password_file"].name == __file__ - assert kwargs["token"] == "tok" - assert kwargs["token_file"].name == __file__ + assert kwargs["certificate"] == "content of a" + assert kwargs["password"] == "content of b" + assert kwargs["token"] == "content of c" assert kwargs["url"] == "https://foo" assert kwargs["username"] == "user" assert kwargs["verify"] is True diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py index 57c614b..076aefc 100644 --- a/tests/unit/test_client.py +++ b/tests/unit/test_client.py @@ -1,5 +1,3 @@ -import io - import pytest from vault_cli import client @@ -96,10 +94,10 @@ def _authenticate_token(self, *args, **kwargs): @pytest.mark.parametrize("test_kwargs, expected", [ ({"token": "yay"}, ["token", "yay"]), ( - {"username": "a", "password": io.BytesIO(b"b")}, + {"username": "a", "password": "b"}, ["userpass", "a", "b"] ), - ({"certificate": io.BytesIO(b"cert")}, ["certificate", "cert"]), + ({"certificate": "cert"}, ["certificate", "cert"]), ]) def test_vault_client_base_authenticate(test_kwargs, expected): auth_params = [] diff --git a/tests/unit/test_clients.py b/tests/unit/test_clients.py index ac86cac..131e00e 100644 --- a/tests/unit/test_clients.py +++ b/tests/unit/test_clients.py @@ -1,5 +1,3 @@ -import io - import pytest from vault_cli import client @@ -42,7 +40,7 @@ def test_userpass(requests_mock, backend): backend=backend, token=None, username="myuser", - password=io.BytesIO(b"pass"), + password="pass", ) # Check that the token is used diff --git a/tests/unit/test_settings.py b/tests/unit/test_settings.py new file mode 100644 index 0000000..9ad423c --- /dev/null +++ b/tests/unit/test_settings.py @@ -0,0 +1,76 @@ +import io + +from vault_cli import settings + + +def test_read_config_file_not_existing(): + assert settings.read_config_file("/non-existant-file") is None + + +def test_read_config_file(tmpdir): + path = tmpdir.join("test.yml") + open(path, "w").write('{"yay": 1}') + + assert settings.read_config_file(path) == {"yay": 1} + + +def test_dash_to_underscores(): + result = settings.dash_to_underscores( + {"a": "b", "c_d": "e_f", "g-h": "i-j"}) + expected = {"a": "b", "c_d": "e_f", "g_h": "i-j"} + assert result == expected + + +def test_read_all_files_no_file(): + d = {"token": "yay", "certificate": "yo", "password": "aaa"} + assert settings.read_all_files(d) == d + + +def test_read_all_files(tmpdir): + open(tmpdir.join("token"), "wb").write(b'yay') + open(tmpdir.join("certificate"), "wb").write(b'yo') + open(tmpdir.join("password"), "wb").write(b'aaa') + + d = {"token_file": tmpdir.join("token"), + "certificate_file": tmpdir.join("certificate"), + "password_file": tmpdir.join("password")} + expected = {"token": "yay", "certificate": "yo", "password": "aaa"} + assert settings.read_all_files(d) == expected + + +def test_read_file_no_path(): + assert settings.read_file(None) is None + + +def test_read_file_stdin(mocker): + mocker.patch("sys.stdin", io.StringIO("yay")) + assert settings.read_file("-") == "yay" + + +def test_build_config_from_files(mocker): + config_file = {"test-a": "b"} + mocker.patch("vault_cli.settings.read_config_file", + return_value=config_file) + read_all_files = mocker.patch("vault_cli.settings.read_all_files", + side_effect=lambda x: x) + + result = settings.build_config_from_files(["a"]) + + assert result["test_a"] == "b" + assert "url" in result + assert read_all_files.called is True + + +def test_build_config_from_files_no_files(mocker): + mocker.patch("vault_cli.settings.read_config_file", + return_value=None) + + result = settings.build_config_from_files(["a"]) + + assert result == settings.DEFAULTS + + +def test_get_vault_options(config): + config.update({"a": "b"}) + + assert settings.get_vault_options(c="d") == {"a": "b", "c": "d"} diff --git a/vault_cli/cli.py b/vault_cli/cli.py index 4adaa7a..ce30eba 100644 --- a/vault_cli/cli.py +++ b/vault_cli/cli.py @@ -32,15 +32,17 @@ default=settings.DEFAULTS['url']) @click.option('--verify/--no-verify', default=settings.DEFAULTS['verify'], help='Verify HTTPS certificate') -@click.option('--certificate', '-c', type=click.File('rb'), - help='Certificate to connect to vault') -@click.option('--token', '-t', help='The token to connect to Vault') -@click.option('--token-file', '-T', type=click.File('rb'), - help='File which contains the token to connect to Vault') +@click.option('--certificate-file', '-c', type=click.Path(), + help='Certificate to connect to vault. ' + 'Configuration file can also contain a "certificate" key.') +@click.option('--token-file', '-T', type=click.Path(), + help='File which contains the token to connect to Vault. ' + 'Configuration file can also contain a "token" key.') @click.option('--username', '-u', help='Username used for userpass authentication') -@click.option('--password-file', '-w', type=click.File('rb'), - help='Can read from stdin if "-" is used as parameter') +@click.option('--password-file', '-w', type=click.Path(), + help='Can read from stdin if "-" is used as parameter. ' + 'Configuration file can also contain a "password" key.') @click.option('--base-path', '-b', help='Base path for requests') @click.option('--backend', default=settings.DEFAULTS['backend'], help='Name of the backend to use (requests, hvac)') @@ -49,9 +51,14 @@ def cli(ctx, **kwargs): Interact with a Vault. See subcommands for details. """ backend = kwargs.pop("backend") + for key in ["password", "certificate", "token"]: + kwargs[key] = settings.CONFIG.get(key) + + # There might still be files to read, so let's do it now + kwargs = settings.read_all_files(kwargs) + try: - ctx.obj = client.get_client_from_kwargs( - backend=backend, **kwargs) + ctx.obj = client.get_client_from_kwargs(backend=backend, **kwargs) except ValueError as exc: raise click.UsageError(str(exc)) diff --git a/vault_cli/settings.py b/vault_cli/settings.py index 76078e5..826145b 100644 --- a/vault_cli/settings.py +++ b/vault_cli/settings.py @@ -16,16 +16,17 @@ limitations under the License. """ -import io import os import yaml +import sys + # Ordered by increasing priority CONFIG_FILES = [ - '/etc/vault.yml', - '~/.vault.yml', './.vault.yml', + '~/.vault.yml', + '/etc/vault.yml', ] DEFAULTS = { @@ -45,18 +46,30 @@ def read_config_file(file_path): with open(os.path.expanduser(file_path), "r") as f: return yaml.safe_load(f) except IOError: - return {} + return None -def clean_config(config): +def dash_to_underscores(config): # Because we're modifying the dict during iteration, we need to # consolidate the keys into a list - for key in list(config): - config[key.replace("-", "_")] = config.pop(key) + return {key.replace("-", "_"): value + for key, value in config.items()} + - config["certificate"] = read_file(config.get("certificate")) - config["password"] = read_file(config.get("password")) - config["token"] = read_file(config.get("token_file")) +def read_all_files(config): + config = config.copy() + # Files override direct values when both are defined + certificate_file = config.pop("certificate_file", None) + if certificate_file: + config["certificate"] = read_file(certificate_file) + + password_file = config.pop("password_file", None) + if password_file: + config["password"] = read_file(password_file) + + token_file = config.pop("token_file", None) + if token_file: + config["token"] = read_file(token_file) return config @@ -65,19 +78,28 @@ def read_file(path): """ Returns the content of the pointed file """ - if path: - with open(os.path.expanduser(path), 'rb') as file_handler: - return file_handler.read().decode("utf-8").strip() + if not path: + return + + if path == "-": + return sys.stdin.read().strip() + + with open(os.path.expanduser(path), 'rb') as file_handler: + return file_handler.read().decode("utf-8").strip() def build_config_from_files(config_files): - config = DEFAULTS.copy() + values = DEFAULTS.copy() for potential_file in config_files: - partial = clean_config(read_config_file(potential_file)) - config.update(partial) + file_config = read_config_file(potential_file) + if file_config is not None: + file_config = dash_to_underscores(file_config) + file_config = read_all_files(file_config) + values.update(file_config) + break - return config + return values # Make sure our config files are not re-read From 4021520534d2e284c4230245e3522c35b36dcf4c Mon Sep 17 00:00:00 2001 From: Joachim Jablon Date: Fri, 28 Sep 2018 17:31:23 +0200 Subject: [PATCH 09/11] Travis --- .gitignore | 2 ++ .travis.yml | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 40c5ffe..0c7e5ee 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,5 @@ build dist .coverage htmlcov +.tox +*.pyc diff --git a/.travis.yml b/.travis.yml index 663a907..896028a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,7 +20,7 @@ matrix: env: TOX_ENV=py27-integration-tests COVERAGE_FLAG=integration before_install: - - "[$COVERAGE_FLAG = integration] && ./dev-env" + - "if [ $COVERAGE_FLAG = integration ]; then ./dev-env; fi" install: - pip install tox codecov From 2562dc6d8bf3d8b80460c0be8cede4bcfae3e31b Mon Sep 17 00:00:00 2001 From: Joachim Jablon Date: Fri, 28 Sep 2018 17:56:28 +0200 Subject: [PATCH 10/11] Fix Python2 --- tests/integration/test_integration.py | 11 +++++------ tests/unit/test_cli.py | 2 +- tests/unit/test_settings.py | 21 +++++++++++++-------- vault_cli/cli.py | 6 +++--- vault_cli/hvac.py | 2 ++ 5 files changed, 24 insertions(+), 18 deletions(-) diff --git a/tests/integration/test_integration.py b/tests/integration/test_integration.py index 16413a0..e777977 100644 --- a/tests/integration/test_integration.py +++ b/tests/integration/test_integration.py @@ -14,26 +14,25 @@ def test_integration_cli(cli_runner): assert call(cli_runner, ["get", "a", "--text"]).output == "b\n" - assert call(cli_runner, ["list"]).output == '''['a']\n''' + assert call(cli_runner, ["list"]).output == "a\n" call(cli_runner, ["set", "c/d", "e"]) - assert call(cli_runner, ["get", "c/d"]).output == "--- e\n...\n\n" + assert call(cli_runner, ["get", "c/d"]).output == "--- e\n...\n" - assert call(cli_runner, ["list"]).output == '''['a', 'c/']\n''' + assert call(cli_runner, ["list"]).output == "a\nc/\n" - assert call(cli_runner, ["list", "c"]).output == '''['d']\n''' + assert call(cli_runner, ["list", "c"]).output == "d\n" assert call(cli_runner, ["get-all", ""]).output == ("""--- a: b c: d: e - """) call(cli_runner, ["delete", "a"]) - assert call(cli_runner, ["list"]).output == '''['c/']\n''' + assert call(cli_runner, ["list"]).output == "c/\n" call(cli_runner, ["delete", "c/d"]) diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index c644fef..bc55ce5 100644 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -78,7 +78,7 @@ def test_options(cli_runner, mocker): def test_list(cli_runner, backend): result = cli_runner.invoke(cli.cli, ["list"]) - assert result.output == "['foo', 'baz']\n" + assert result.output == "foo\nbaz\n" assert result.exit_code == 0 diff --git a/tests/unit/test_settings.py b/tests/unit/test_settings.py index 9ad423c..b3148e9 100644 --- a/tests/unit/test_settings.py +++ b/tests/unit/test_settings.py @@ -1,3 +1,5 @@ +from __future__ import unicode_literals + import io from vault_cli import settings @@ -8,7 +10,7 @@ def test_read_config_file_not_existing(): def test_read_config_file(tmpdir): - path = tmpdir.join("test.yml") + path = str(tmpdir.join("test.yml")) open(path, "w").write('{"yay": 1}') assert settings.read_config_file(path) == {"yay": 1} @@ -27,13 +29,16 @@ def test_read_all_files_no_file(): def test_read_all_files(tmpdir): - open(tmpdir.join("token"), "wb").write(b'yay') - open(tmpdir.join("certificate"), "wb").write(b'yo') - open(tmpdir.join("password"), "wb").write(b'aaa') - - d = {"token_file": tmpdir.join("token"), - "certificate_file": tmpdir.join("certificate"), - "password_file": tmpdir.join("password")} + token_path = str(tmpdir.join("token")) + open(token_path, "wb").write(b'yay') + certificate_path = str(tmpdir.join("certificate")) + open(certificate_path, "wb").write(b'yo') + password_path = str(tmpdir.join("password")) + open(password_path, "wb").write(b'aaa') + + d = {"token_file": token_path, + "certificate_file": certificate_path, + "password_file": password_path} expected = {"token": "yay", "certificate": "yo", "password": "aaa"} assert settings.read_all_files(d) == expected diff --git a/vault_cli/cli.py b/vault_cli/cli.py index ce30eba..374bc78 100644 --- a/vault_cli/cli.py +++ b/vault_cli/cli.py @@ -72,7 +72,7 @@ def list_(client_obj, path): is given, list the objects at the root. """ result = client_obj.list_secrets(path=path) - click.echo(result) + click.echo("\n".join(result)) @cli.command(name='get-all') @@ -91,7 +91,7 @@ def get_all(client_obj, path): click.echo(yaml.safe_dump( result, default_flow_style=False, - explicit_start=True)) + explicit_start=True), nl=False) @cli.command() @@ -112,7 +112,7 @@ def get(client_obj, text, name): click.echo(yaml.safe_dump(secret, default_flow_style=False, - explicit_start=True)) + explicit_start=True), nl=False) @cli.command("set") diff --git a/vault_cli/hvac.py b/vault_cli/hvac.py index 751cd18..a5d2559 100644 --- a/vault_cli/hvac.py +++ b/vault_cli/hvac.py @@ -16,6 +16,8 @@ limitations under the License. """ +from __future__ import absolute_import + import hvac from vault_cli.client import VaultAPIException From 9816ca77a53d1edffaf0cce8f7aa73994928bed3 Mon Sep 17 00:00:00 2001 From: Joachim Jablon Date: Fri, 28 Sep 2018 18:15:25 +0200 Subject: [PATCH 11/11] Update readme --- README.md | 54 ++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 38 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 401e929..802bcbe 100644 --- a/README.md +++ b/README.md @@ -25,18 +25,20 @@ Usage: vault [OPTIONS] COMMAND [ARGS]... Interact with a Vault. See subcommands for details. Options: - -U, --url TEXT URL of the vault instance - --verify / --no-verify Verify HTTPS certificate - -c, --certificate FILENAME The certificate to connect to vault - -t, --token TEXT The token to connect to Vault - -T, --token-file FILENAME File which contains the token to connect to - Vault - -u, --username TEXT The username used for userpass authentication - -w, --password-file FILENAME Can read from stdin if "-" is used as - parameter - -b, --base-path TEXT Base path for requests - --backend TEXT Name of the backend to use (requests, hvac) - -h, --help Show this message and exit. + -U, --url TEXT URL of the vault instance + --verify / --no-verify Verify HTTPS certificate + -c, --certificate-file PATH Certificate to connect to vault. Configuration + file can also contain a "certificate" key. + -T, --token-file PATH File which contains the token to connect to + Vault. Configuration file can also contain a + "token" key. + -u, --username TEXT Username used for userpass authentication + -w, --password-file PATH Can read from stdin if "-" is used as + parameter. Configuration file can also contain + a "password" key. + -b, --base-path TEXT Base path for requests + --backend TEXT Name of the backend to use (requests, hvac) + -h, --help Show this message and exit. Commands: delete Deletes a single secret. @@ -45,6 +47,7 @@ Commands: list List all the secrets at the given path. set Set a single secret to the given value(s). + ``` ## Authentication @@ -108,14 +111,13 @@ Done ## Configuration -All files at the following location are read (in increasing priority order), -parsed, merged and used: +The first file found in the following location is read, parsed and used: 1. `/etc/vault.yml` 2. `~/.vault.yml` 3. `./.vault.yml` Any option passed as command line flag will be used over the corresponding -option in the documentation. +option in the documentation (use either `-` or `_`). The expected format of the configuration is a mapping, with option names and their corresponding values: @@ -132,12 +134,32 @@ base-path: project/ ... ``` +Make sure the secret files have their permissions set accordingly. + +For simple cases, you can directly define your `token` or `password` in the +file: + +```yaml +--- +username: my_username +password: secret-password +# or +token: secret-token +url: https://vault.mydomain:8200 +verify: no +base-path: project/ +... +``` + +If you do so, make sure the permissions of the configuration file itself are +not too broad + Just note that the `--verify / --no-verify` flag become `verify: yes` or `verify: no` ## State -The tool is currently in beta mode. It's missing docs, tests, CI, and such. +The tool is currently in beta mode. It's missing docs, linting, and such. Be warned. ## License