From dada892ee354ef80da586201014ed1e288295d6b Mon Sep 17 00:00:00 2001 From: Jacob Callahan Date: Mon, 7 Oct 2024 16:27:29 -0400 Subject: [PATCH 1/4] Rework inventory cli With the changes we've been making in 0.6 toward a better, more customizable and visually appealing, user experience. It's time that we revisited the inventory command. The --curated view introduced in 0.5.x has now become the default inventory view. The original default view is now present in its new form and home with the --list flag. The --details flag largely retains the functionality it did before. However, it involves much less processing than before. One important piece added in this is the ability to customize the inventory fields on a per-user basis. The details on this customization are documented in detail in the supporting code. It will also be important to document this in the wiki upon release. --- broker/commands.py | 63 +++++++------- broker/config_migrations/v0_6_0.py | 25 ++++++ broker/helpers.py | 133 +++++++++++++++++++++++------ broker/settings.py | 2 + broker_settings.yaml.example | 9 ++ 5 files changed, 174 insertions(+), 58 deletions(-) diff --git a/broker/commands.py b/broker/commands.py index fd41461..a51ff94 100644 --- a/broker/commands.py +++ b/broker/commands.py @@ -285,49 +285,52 @@ def checkin(hosts, background, all_, sequential, filter): @loggedcli() @click.option("--details", is_flag=True, help="Display all host details") -@click.option("--curated", is_flag=True, help="Display curated host details") +@click.option("--list", "_list", is_flag=True, help="Display only hostnames and local ids") @click.option( "--sync", type=str, help="Class-style name of a supported broker provider. (AnsibleTower)", ) @click.option("--filter", type=str, help="Display only what matches the specified filter") -def inventory(details, curated, sync, filter): - """Get a list of all hosts you've checked out showing hostname and local id. +def inventory(details, _list, sync, filter): + """Display a table of hosts in your local inventory. - hostname pulled from list of dictionaries. + Inventory fields are configurable in Broker's settings file. + + Run a sync for your providers to pull down your host information. + + e.g. `broker inventory --sync AnsibleTower` + + Note: Applying a filter will result in incorrect id's being displayed. """ if sync: Broker.sync_inventory(provider=sync) - logger.info("Pulling local inventory") inventory = helpers.load_inventory(filter=filter) helpers.emit({"inventory": inventory}) - if curated: - table = Table(title="Host Inventory") - - table.add_column("Id", justify="left", style="cyan", no_wrap=True) - table.add_column("Host", justify="left", style="magenta") - table.add_column("Provider", justify="left", style="green") - table.add_column("Action", justify="left", style="yellow") - table.add_column("OS", justify="left", style="blue") - - for host in helpers.get_host_inventory_fields(inventory, PROVIDER_ACTIONS): - table.add_row( - str(host["id"]), host["host"], host["provider"], host["action"], host["os"] - ) - - CONSOLE.print(table) + # details is handled differently than the normal and list views + if details: + detailed = helpers.yaml_format(dict(enumerate(inventory))) + CONSOLE.print(Syntax(detailed, "yaml", background_color="default")) return - for num, host in enumerate(inventory): - if (display_name := host.get("hostname")) is None: - display_name = host.get("name") - # if we're filtering, then don't show an index. - # Otherwise, a user might perform an action on the incorrect (unfiltered) index. - index = f"{num}: " if filter is None else "" - if details: - logger.info(f"{index}{display_name}:\n{helpers.yaml_format(host)}") - else: - logger.info(f"{index}{display_name}") + + inventory_fields = ( + {"Host": settings.settings.inventory_list_vars} + if _list + else settings.settings.inventory_fields + ) + curated_host_info = [ + helpers.inventory_fields_to_dict( + inventory_fields=inventory_fields, + host_dict=host, + provider_actions=PROVIDER_ACTIONS, + ) + for host in inventory + ] + table = helpers.dictlist_to_table(curated_host_info, "Host Inventory", _id=True) + if _list: + table.title = None + table.box = None + CONSOLE.print(table) @loggedcli() diff --git a/broker/config_migrations/v0_6_0.py b/broker/config_migrations/v0_6_0.py index a1cbce2..d1eb537 100644 --- a/broker/config_migrations/v0_6_0.py +++ b/broker/config_migrations/v0_6_0.py @@ -1,4 +1,5 @@ """Config migrations for versions older than 0.6.0 to 0.6.0.""" + from logzero import logger TO_VERSION = "0.6.0" @@ -60,6 +61,29 @@ def add_thread_limit(config_dict): return config_dict +def add_inventory_fields(config_dict): + """Inventory fields are new in this version. + + Example: + # Customize the fields and values presented by `broker inventory` + # Almost all field values should correspond to a field in your Broker inventory + inventory_fields: + Host: hostname | name # use a | to allow fallback values + Provider: _broker_provider # just pull the _broker_provider value + Action: $action # some special field values are possible, check the wiki + OS: os_distribution os_distribution_version # you can combine multiple values with a space between + """ + logger.debug("Adding inventory fields to the config.") + config_dict["inventory_fields"] = { + "Host": "hostname", + "Provider": "_broker_provider", + "Action": "$action", + "OS": "os_distribution os_distribution_version", + } + config_dict["inventory_list_vars"] = "hostname | name" + return config_dict + + def run_migrations(config_dict): """Run all migrations.""" logger.info(f"Running config migrations for {TO_VERSION}.") @@ -68,5 +92,6 @@ def run_migrations(config_dict): config_dict = remove_test_nick(config_dict) config_dict = move_ssh_settings(config_dict) config_dict = add_thread_limit(config_dict) + config_dict = add_inventory_fields(config_dict) config_dict["_version"] = TO_VERSION return config_dict diff --git a/broker/helpers.py b/broker/helpers.py index b65ae55..6d1e32f 100644 --- a/broker/helpers.py +++ b/broker/helpers.py @@ -19,6 +19,7 @@ import click from logzero import logger +from rich.table import Table from ruamel.yaml import YAML from broker import exceptions, logger as b_log, settings @@ -30,6 +31,18 @@ yaml.default_flow_style = False yaml.sort_keys = False +SPECIAL_INVENTORY_FIELDS = {} # use the _special_inventory_field decorator to add new fields + + +def _special_inventory_field(action_name): + """Register inventory field actions.""" + + def decorator(func): + SPECIAL_INVENTORY_FIELDS[action_name] = func + return func + + return decorator + def clean_dict(in_dict): """Remove entries from a dict where value is None.""" @@ -98,7 +111,7 @@ def flatten_dict(nested_dict, parent_key="", separator="_"): return dict(flattened) -def dict_from_paths(source_dict, paths): +def dict_from_paths(source_dict, paths, sep="/"): """Given a dictionary of desired keys and nested paths, return a new dictionary. Example: @@ -122,10 +135,10 @@ def dict_from_paths(source_dict, paths): """ result = {} for key, path in paths.items(): - if "/" not in path: + if sep not in path: result[key] = source_dict.get(path) else: - top, rem = path.split("/", 1) + top, rem = path.split(sep, 1) result.update(dict_from_paths(source_dict[top], {key: rem})) return result @@ -279,32 +292,73 @@ def flip_provider_actions(provider_actions): return flipped -def get_host_inventory_fields(inv_dict, provider_actions): +def inventory_fields_to_dict(inventory_fields, host_dict, **extras): + """Convert a dicionary-like representation of inventory fields to a resolved dictionary. + + inventory fields, as set in the config look like this, in yaml: + inventory_fields: + Host: hostname | name + Provider: _broker_provider + Action: $action + OS: os_distribution os_distribution_version + + We then process that into a dictionary with inventory values like this: + { + "Host": "some.test.host", + "Provider": "AnsibleTower", + "Action": "deploy-base-rhel", + "OS": "RHEL 8.4" + } + + Notes: The special syntax use in Host and Action fields <$action> is a special keyword that + represents a more complex field resolved by Broker. + Also, the Host field represents a priority order of single values, + so if hostname is not present, name will be used. + Finally, spaces between values are preserved. This lets us combine multiple values in a single field. + """ + return { + name: _resolve_inv_field(field, host_dict, **extras) + for name, field in inventory_fields.items() + } + + +def _resolve_inv_field(field, host_dict, **extras): + """Real functionality for inventory_fields_to_dict, allows recursive evaluation.""" + # Users can specify multiple values to try in order of priority, so evaluate each + if "|" in field: + resolved = [_resolve_inv_field(f.strip(), host_dict, **extras) for f in field.split("|")] + for val in resolved: + if val: + return val + return "Unknown" + # Users can combine multiple values in a single field, so evaluate each + if " " in field: + return " ".join(_resolve_inv_field(f, host_dict, **extras) for f in field.split()) + # Some field values require special handling beyond what the existing syntax allows + if special_field_func := SPECIAL_INVENTORY_FIELDS.get(field): + return special_field_func(host_dict, **extras) + # Otherwise, try to get the value from the host dictionary + return dict_from_paths(host_dict, {"_": field}, sep=".")["_"] or "Unknown" + + +@_special_inventory_field("$action") +def get_host_action(host_dict, provider_actions=None, **_): """Get a more focused set of fields from the host inventory.""" - flipped_prov_actions = flip_provider_actions(provider_actions) - curated_hosts = [] - for num, host in enumerate(inv_dict): - match host: - case { - "name": name, - "hostname": hostname, - "_broker_provider": provider, - }: - os_name = host.get("os_distribution", "Unknown") - os_version = host.get("os_distribution_version", "") - for opt in flipped_prov_actions[provider]: - if action := host["_broker_args"].get(opt): - curated_hosts.append( - { - "id": num, - "host": hostname or name, - "provider": provider, - "os": f"{os_name} {os_version}", - "action": action, - } - ) - break - return curated_hosts + if not provider_actions: + return "$actionError" + # Flip the mapping of actions->provider to provider->actions + flipped_actions = {} + for action, (provider, _) in provider_actions.items(): + provider_name = provider.__name__ + if provider_name not in flipped_actions: + flipped_actions[provider_name] = [] + flipped_actions[provider_name].append(action) + # Get the host's action, based on its provider + provider = host_dict["_broker_provider"] + for opt in flipped_actions[provider]: + if action := host_dict["_broker_args"].get(opt): + return action + return "Unknown" def kwargs_from_click_ctx(ctx): @@ -657,3 +711,26 @@ def temporary_tar(paths): tar.add(path, arcname=path.name) yield temp_tar.absolute() temp_tar.unlink() + + +def dictlist_to_table(dict_list, title=None, _id=False): + """Convert a list of dictionaries to a rich table.""" + # I like pretty colors, so let's cycle through them + column_colors = ["cyan", "magenta", "green", "yellow", "blue", "red"] + curr_color = 0 + table = Table(title=title) + # construct the columns + if _id: # likely just for inventory tables + table.add_column("Id", justify="left", style=column_colors[curr_color], no_wrap=True) + curr_color += 1 + for key in dict_list[0]: # assume all dicts have the same keys + table.add_column(key, justify="left", style=column_colors[curr_color]) + curr_color += 1 + if curr_color >= len(column_colors): + curr_color = 0 + # add the rows + for id_num, data_dict in enumerate(dict_list): + row = [str(id_num)] if _id else [] + row.extend([str(value) for value in data_dict.values()]) + table.add_row(*row) + return table diff --git a/broker/settings.py b/broker/settings.py index 20101e6..ae2edb5 100644 --- a/broker/settings.py +++ b/broker/settings.py @@ -76,6 +76,8 @@ default="debug", ), Validator("THREAD_LIMIT", default=None), + Validator("INVENTORY_FIELDS", is_type_of=dict), + Validator("INVENTORY_LIST_VARS", is_type_of=str, default="hostname | name"), ] # temporary fix for dynaconf #751 diff --git a/broker_settings.yaml.example b/broker_settings.yaml.example index 747d384..84ff0b8 100644 --- a/broker_settings.yaml.example +++ b/broker_settings.yaml.example @@ -4,6 +4,15 @@ _version: 0.6.0 logging: console_level: info file_level: debug +# Customize the fields and values presented by `broker inventory` +# Almost all field values should correspond to a field in your Broker inventory +inventory_fields: + Host: hostname | name # use a | to allow fallback values + Provider: _broker_provider # just pull the _broker_provider value + Action: $action # some special field values are possible, check the wiki + OS: os_distribution os_distribution_version # you can combine multiple values with a space between +# Much like you can set a variable lookup order for inventory fields +inventory_list_vars: hostname | name | ip # Optionally set a limit for the number of threads Broker can use for actions thread_limit: None # Host SSH Settings From e2647995f1f296e251ef2e8b5aa5a0d08c1fbd6f Mon Sep 17 00:00:00 2001 From: Jacob Callahan Date: Tue, 8 Oct 2024 09:34:00 -0400 Subject: [PATCH 2/4] Add in the sad ability to lessen color output for rich In case people want to have a more plain output. --- broker/commands.py | 2 +- broker/config_migrations/v0_6_0.py | 8 ++++++++ broker/settings.py | 1 + broker_settings.yaml.example | 2 ++ 4 files changed, 12 insertions(+), 1 deletion(-) diff --git a/broker/commands.py b/broker/commands.py index a51ff94..181e157 100644 --- a/broker/commands.py +++ b/broker/commands.py @@ -17,7 +17,7 @@ from broker.providers import PROVIDER_ACTIONS, PROVIDER_HELP, PROVIDERS signal.signal(signal.SIGINT, helpers.handle_keyboardinterrupt) -CONSOLE = Console() # rich console for pretty printing +CONSOLE = Console(no_color=settings.settings.less_colors) # rich console for pretty printing click.rich_click.SHOW_ARGUMENTS = True click.rich_click.COMMAND_GROUPS = { diff --git a/broker/config_migrations/v0_6_0.py b/broker/config_migrations/v0_6_0.py index d1eb537..080f4d2 100644 --- a/broker/config_migrations/v0_6_0.py +++ b/broker/config_migrations/v0_6_0.py @@ -84,6 +84,13 @@ def add_inventory_fields(config_dict): return config_dict +def add_color_control(config_dict): + """Add in the new `less_colors` field.""" + logger.debug("Adding the less_colors field to the config.") + config_dict["less_colors"] = config_dict.get("less_colors", False) + return config_dict + + def run_migrations(config_dict): """Run all migrations.""" logger.info(f"Running config migrations for {TO_VERSION}.") @@ -93,5 +100,6 @@ def run_migrations(config_dict): config_dict = move_ssh_settings(config_dict) config_dict = add_thread_limit(config_dict) config_dict = add_inventory_fields(config_dict) + config_dict = add_color_control(config_dict) config_dict["_version"] = TO_VERSION return config_dict diff --git a/broker/settings.py b/broker/settings.py index ae2edb5..9a52815 100644 --- a/broker/settings.py +++ b/broker/settings.py @@ -78,6 +78,7 @@ Validator("THREAD_LIMIT", default=None), Validator("INVENTORY_FIELDS", is_type_of=dict), Validator("INVENTORY_LIST_VARS", is_type_of=str, default="hostname | name"), + Validator("LESS_COLORS", default=False), ] # temporary fix for dynaconf #751 diff --git a/broker_settings.yaml.example b/broker_settings.yaml.example index 84ff0b8..f6cf3eb 100644 --- a/broker_settings.yaml.example +++ b/broker_settings.yaml.example @@ -1,5 +1,7 @@ # Broker settings _version: 0.6.0 +# Disable rich colors +less_colors: False # different log levels for file and stdout logging: console_level: info From 6c3797a0bc4c1c7011eb5635f1e840ce0c5efc8a Mon Sep 17 00:00:00 2001 From: Jacob Callahan Date: Tue, 8 Oct 2024 11:23:40 -0400 Subject: [PATCH 3/4] Adjust a couple of migrations I had forgot the include logic to not run some migrations if the changes have already taken place. This skip is just the "naive" approach instead of doing a complete validation that the previous fields have migrated. --- broker/config_migrations/v0_6_0.py | 9 +++++++-- broker/helpers.py | 2 +- broker_settings.yaml.example | 4 ++-- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/broker/config_migrations/v0_6_0.py b/broker/config_migrations/v0_6_0.py index 080f4d2..72e464c 100644 --- a/broker/config_migrations/v0_6_0.py +++ b/broker/config_migrations/v0_6_0.py @@ -37,7 +37,10 @@ def remove_test_nick(config_dict): def move_ssh_settings(config_dict): - """Move SSH settings from the top leve into its own chunk.""" + """Move SSH settings from the top level into its own chunk.""" + # Check if the migration has already been performed + if "ssh" in config_dict: + return config_dict logger.debug("Moving SSH settings into their own section.") ssh_settings = { "backend": config_dict.pop("ssh_backend", "ssh2-python312"), @@ -57,7 +60,7 @@ def move_ssh_settings(config_dict): def add_thread_limit(config_dict): """Add a thread limit to the config.""" logger.debug("Adding a thread limit to the config.") - config_dict["thread_limit"] = None + config_dict["thread_limit"] = config_dict.get("thread_limit") return config_dict @@ -73,6 +76,8 @@ def add_inventory_fields(config_dict): Action: $action # some special field values are possible, check the wiki OS: os_distribution os_distribution_version # you can combine multiple values with a space between """ + if "inventory_fields" in config_dict: + return config_dict logger.debug("Adding inventory fields to the config.") config_dict["inventory_fields"] = { "Host": "hostname", diff --git a/broker/helpers.py b/broker/helpers.py index 6d1e32f..707fe9f 100644 --- a/broker/helpers.py +++ b/broker/helpers.py @@ -328,7 +328,7 @@ def _resolve_inv_field(field, host_dict, **extras): if "|" in field: resolved = [_resolve_inv_field(f.strip(), host_dict, **extras) for f in field.split("|")] for val in resolved: - if val: + if val and val != "Unknown": return val return "Unknown" # Users can combine multiple values in a single field, so evaluate each diff --git a/broker_settings.yaml.example b/broker_settings.yaml.example index f6cf3eb..8c98157 100644 --- a/broker_settings.yaml.example +++ b/broker_settings.yaml.example @@ -16,7 +16,7 @@ inventory_fields: # Much like you can set a variable lookup order for inventory fields inventory_list_vars: hostname | name | ip # Optionally set a limit for the number of threads Broker can use for actions -thread_limit: None +thread_limit: null # Host SSH Settings # These can be left alone if you're not using Broker as a library ssh: @@ -49,7 +49,7 @@ Container: docker: host_username: "" host_password: "" - host_port: None + host_port: null runtime: docker network: null default: True From 7184a7b60639a81ef289061dcbf333210da8b09f Mon Sep 17 00:00:00 2001 From: Jake Callahan Date: Thu, 10 Oct 2024 16:26:29 -0400 Subject: [PATCH 4/4] Update broker/helpers.py Co-authored-by: Tasos Papaioannou --- broker/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/broker/helpers.py b/broker/helpers.py index 707fe9f..77f2ddd 100644 --- a/broker/helpers.py +++ b/broker/helpers.py @@ -306,7 +306,7 @@ def inventory_fields_to_dict(inventory_fields, host_dict, **extras): { "Host": "some.test.host", "Provider": "AnsibleTower", - "Action": "deploy-base-rhel", + "Action": "deploy-rhel", "OS": "RHEL 8.4" }