diff --git a/riocli/config/config.py b/riocli/config/config.py index 68e8a499..db5d1545 100644 --- a/riocli/config/config.py +++ b/riocli/config/config.py @@ -45,6 +45,7 @@ class Configuration(object): APP_NAME = 'rio-cli' PIPING_SERVER = 'https://piping-server-v0-rapyuta-infra.apps.okd4v2.okd4beta.rapyuta.io' DIFF_TOOL = 'diff' + MERGE_TOOL = 'vimdiff' def __init__(self, filepath: Optional[str] = None): self._filepath = filepath @@ -171,6 +172,10 @@ def piping_server(self: Configuration): def diff_tool(self: Configuration): return self.data.get('diff_tool', self.DIFF_TOOL) + @property + def merge_tool(self: Configuration): + return self.data.get('merge_tool', self.MERGE_TOOL) + @property def machine_id(self: Configuration): if 'machine_id' not in self.data: diff --git a/riocli/configtree/__init__.py b/riocli/configtree/__init__.py index 48929b1d..53e2935b 100644 --- a/riocli/configtree/__init__.py +++ b/riocli/configtree/__init__.py @@ -16,6 +16,7 @@ from riocli.configtree.diff import diff_revisions from riocli.configtree.export_keys import export_keys +from riocli.configtree.merge import merge_revisions from riocli.configtree.revision import revision from riocli.configtree.tree import clone_tree, create_config_tree, delete_config_tree, list_config_tree_keys, list_config_trees, list_tree_revisions, set_tree_revision from riocli.configtree.import_keys import import_keys @@ -46,3 +47,4 @@ def config_trees() -> None: config_trees.add_command(list_tree_revisions) config_trees.add_command(revision) config_trees.add_command(diff_revisions) +config_trees.add_command(merge_revisions) diff --git a/riocli/configtree/diff.py b/riocli/configtree/diff.py index c4b42b7f..ccf46f81 100644 --- a/riocli/configtree/diff.py +++ b/riocli/configtree/diff.py @@ -30,6 +30,31 @@ @click.argument('ref_2', type=str) @click.pass_context def diff_revisions(ctx: click.Context, ref_1: str, ref_2: str): + """ + Diff between two revisions of the same or different trees. + + The ref is a slash ('/') separated string. + + * The first part can be 'org' or 'proj' defining the scope of the reference. + + * The second part defines the name of the Tree. + + * The third optional part defines the revision-id or milestone of the Tree. + + Examples: + + * org/tree-name + + * org/tree-name/rev-id + + * org/tree-name/milestone + + * proj/tree-name + + * proj/tree-name/rev-id + + * proj/tree-name/milestone + """ try: ref_1_keys = fetch_ref_keys(ref_1) ref_2_keys = fetch_ref_keys(ref_2) diff --git a/riocli/configtree/merge.py b/riocli/configtree/merge.py new file mode 100644 index 00000000..aa0f5be6 --- /dev/null +++ b/riocli/configtree/merge.py @@ -0,0 +1,178 @@ +# Copyright 2024 Rapyuta Robotics +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +from tempfile import NamedTemporaryFile +from typing import Optional +from benedict import benedict +import click +from click_help_colors import HelpColorsCommand +from yaspin.core import Yaspin + +from riocli.config import get_config_from_context, new_v2_client +from riocli.configtree.import_keys import split_metadata +from riocli.configtree.revision import Revision +from riocli.configtree.util import Metadata, combine_metadata, fetch_last_milestone_keys, fetch_ref_keys, fetch_tree_keys, unflatten_keys +from riocli.constants.colors import Colors +from riocli.constants.symbols import Symbols +from riocli.utils.spinner import with_spinner + + +@click.command( + 'merge', + cls=HelpColorsCommand, + help_headers_color=Colors.YELLOW, + help_options_color=Colors.GREEN, +) +@click.argument('base-tree-name', type=str) +@click.argument('ref', type=str) +@click.option('--silent', 'silent', is_flag=True, type=click.BOOL, default=False, + help='Skip interactively, if fast merge is not possible, then fail.') +@click.option('--ignore-conflict', 'ignore_conflict', is_flag=True, type=click.BOOL, + default=False, help='Skip the conflicting keys and only perform a partial fast merge.') +@click.option('--milestone', 'milestone', type=str, + help='Minestone name for the imported revision.') +@click.option('--organization', 'with_org', is_flag=True, type=bool, + default=False, help='Operate on organization-scoped Config Trees only.') +@click.pass_context +@with_spinner(text="Merging...") +def merge_revisions(ctx: click.Context, base_tree_name: str, ref: str, + silent: bool, with_org: bool, milestone: Optional[str], + ignore_conflict: bool, spinner: Yaspin): + """ + Merge the revision specified by the ref on the base-tree. The Base tree must + be the name of the tree. Merge always works on the HEAD of the tree. + + The ref is a slash ('/') separated string. + + * The first part can be 'org' or 'proj' defining the scope of the reference. + + * The second part defines the name of the Tree. + + * The third optional part defines the revision-id or milestone of the Tree. + + Examples: + + * org/tree-name + + * org/tree-name/rev-id + + * org/tree-name/milestone + + * proj/tree-name + + * proj/tree-name/rev-id + + * proj/tree-name/milestone + + """ + try: + base_keys = fetch_tree_keys(is_org=with_org, tree_name=base_tree_name) + source_keys = fetch_ref_keys(ref=ref) + old_base_keys = fetch_last_milestone_keys(is_org=with_org, tree_name=base_tree_name) + fast_merge_possible = is_fast_merge_possible(base_keys, source_keys) + + if not fast_merge_possible and not ignore_conflict and silent: + raise Exception('Fast merge is not possible') + + fast_merge(base_keys, source_keys) + + if not ignore_conflict and not fast_merge_possible: + with spinner.hidden(): + merged = interactive_merge(ctx, base_keys, source_keys, old_base_keys) + data, metadata = split_metadata(merged) + else: + spinner.write(click.style('{} Fast-forwarding'.format(Symbols.INFO), fg=Colors.CYAN)) + # Combining and Splitting is required to remove the extra API fields + # in the Value. + base_keys = combine_metadata(base_keys) + data, metadata = split_metadata(base_keys) + + data = benedict(data).flatten(separator='/') + metadata = benedict(metadata).flatten(separator='/') + + client = new_v2_client(with_project=(not with_org)) + with Revision(tree_name=base_tree_name, client=client, force_new=True, + with_org=with_org, commit=True, milestone=milestone) as rev: + rev_id = rev.revision_id + + for key, value in data.items(): + key_metadata = metadata.get(key, None) + if key_metadata is not None and isinstance(key_metadata, Metadata): + key_metadata = key_metadata.get_dict() + + rev.store(key=key, value=str(value), perms=644, metadata=key_metadata) + + payload = { + 'kind': 'ConfigTree', + 'apiVersion': 'api.rapyuta.io/v2', + 'metadata': { + 'name': base_tree_name, + }, + 'head': { + 'metadata': { + 'guid': rev_id, + } + }, + } + + client.set_revision_config_tree(base_tree_name, payload) + spinner.text = click.style('Config tree merged successfully.', fg=Colors.CYAN) + spinner.green.ok(Symbols.SUCCESS) + except Exception as e: + spinner.red.text = str(e) + spinner.red.fail(Symbols.ERROR) + raise SystemExit(1) from e + + +def interactive_merge(ctx: click.Context, keys_1: dict, keys_2: dict, keys_3: Optional[dict]) -> dict: + cfg = get_config_from_context(ctx) + keys_1 = unflatten_keys(keys_1) + keys_2 = unflatten_keys(keys_2) + keys_3 = unflatten_keys(keys_3) + + with NamedTemporaryFile(mode='w+b', prefix='HEAD_') as file_1, \ + NamedTemporaryFile(mode='w+b', prefix='MERGE_HEAD_') as file_2, \ + NamedTemporaryFile(mode='w+b', prefix='PREV_BASE_') as file_3: + + keys_1.to_json(filepath=file_1.name, indent=4) + keys_2.to_json(filepath=file_2.name, indent=4) + keys_3.to_json(filepath=file_3.name, indent=4) + os.system('{} {} {} {}'.format(cfg.merge_tool, file_3.name, file_1.name, file_2.name)) + + return benedict(file_1.name, format='json') + + +def is_fast_merge_possible(base: dict, source: dict) -> bool: + for key, value in base.items(): + source_value = source.get(key) + if not source_value: + continue + + source_data, base_data = source_value.get('data'), value.get('data') + if source_data != base_data: + return False + + source_meta, base_meta = source_value.get('metadata'), value.get('metadata') + if source_meta != base_meta: + return False + + return True + + +def fast_merge(base: dict, source: dict) -> None: + for key, value in source.items(): + if not base.get(key): + base[key] = value + + diff --git a/riocli/configtree/util.py b/riocli/configtree/util.py index 50c0922b..79b5e7f9 100644 --- a/riocli/configtree/util.py +++ b/riocli/configtree/util.py @@ -187,7 +187,7 @@ def parse_ref(input: str) -> (bool, str, Optional[str]): * The first part can be 'org' or 'proj' defining the scope of the reference. * The second part defines the name of the Tree. - * The third optional part defines the revision-id of the Tree. + * The third optional part defines the revision-id or milestone of the Tree. This function returns a Tuple with 3 values: @@ -210,17 +210,14 @@ def parse_ref(input: str) -> (bool, str, Optional[str]): if len(splits) < 2: raise Exception('ref {} is invalid'.format(input)) - if splits[0] == 'org': - is_org = True - elif splits[0] == 'proj': - is_org = False - else: + if splits[0] not in ('org', 'proj'): raise Exception('ref scope {} is invalid'.format(splits[0])) + is_org = splits[0] == 'org' + + revision = None if len(splits) == 3: revision = splits[2] - else: - revision = None milestone = None if revision is not None and not revision.startswith('rev-'): @@ -230,7 +227,10 @@ def parse_ref(input: str) -> (bool, str, Optional[str]): return is_org, splits[1], revision, milestone -def unflatten_keys(keys: dict) -> benedict: +def unflatten_keys(keys: Optional[dict]) -> benedict: + if keys is None: + return benedict() + data = combine_metadata(keys) return benedict(data).unflatten(separator='/') @@ -242,6 +242,9 @@ def combine_metadata(keys: dict) -> dict: data = val.get('data', None) if data is not None: data = b64decode(data).decode('utf-8') + # The data received from the API is always in string format. To use + # appropriate data-type in Python (as well in exports), we are + # passing it through YAML parser. data = yaml.safe_load(data) metadata = val.get('metadata', None) @@ -253,6 +256,19 @@ def combine_metadata(keys: dict) -> dict: return result +def fetch_last_milestone_keys(is_org: bool, tree_name: str) -> Optional[dict]: + client = new_v2_client(with_project=(not is_org)) + revisions = client.list_config_tree_revisions(tree_name=tree_name) + if len(revisions) == 0: + return + + for rev in revisions: + milestone = get_revision_milestone(rev) + if milestone is not None: + return fetch_tree_keys(is_org=is_org, tree_name=tree_name, + rev_id=rev.metadata.guid) + + def fetch_ref_keys(ref: str) -> dict: is_org, tree, rev_id, milestone = parse_ref(ref) return fetch_tree_keys(is_org=is_org, tree_name=tree, rev_id=rev_id,