Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

EdkRepo: Update the Sync Command to Support Local Directory Renames and URL Changes #5

Merged
merged 2 commits into from
Jun 4, 2021
Merged

EdkRepo: Update the Sync Command to Support Local Directory Renames and URL Changes #5

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
fc01c03
EdkRepo: Correct Sync Print Formatting
ashedesimone May 15, 2021
87eee2d
EdkRepo: Sync -u should support repository Local directory renames an…
ashedesimone May 15, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
63 changes: 36 additions & 27 deletions edkrepo/commands/sync_command.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
## @file
# sync_command.py
#
# Copyright (c) 2017 - 2020, Intel Corporation. All rights reserved.<BR>
# Copyright (c) 2017 - 2021, Intel Corporation. All rights reserved.<BR>
# SPDX-License-Identifier: BSD-2-Clause-Patent
#

Expand All @@ -30,9 +30,9 @@
from edkrepo.common.humble import NO_SYNC_DETACHED_HEAD, SYNC_COMMITS_ON_MASTER, SYNC_ERROR
from edkrepo.common.humble import MIRROR_BEHIND_PRIMARY_REPO, SYNC_NEEDS_REBASE, INCLUDED_FILE_NAME
from edkrepo.common.humble import SYNC_BRANCH_CHANGE_ON_LOCAL, SYNC_INCOMPATIBLE_COMBO
from edkrepo.common.humble import SYNC_REBASE_CALC_FAIL
from edkrepo.common.humble import SYNC_REBASE_CALC_FAIL, SYNC_MOVE_FAILED
from edkrepo.common.pathfix import get_actual_path, expanduser
from edkrepo.common.common_cache_functions import get_repo_cache_obj
from edkrepo.common.common_cache_functions import get_repo_cache_obj
from edkrepo.common.common_repo_functions import clone_repos, sparse_checkout_enabled
from edkrepo.common.common_repo_functions import reset_sparse_checkout, sparse_checkout, verify_single_manifest
from edkrepo.common.common_repo_functions import checkout_repos, check_dirty_repos
Expand All @@ -41,15 +41,15 @@
from edkrepo.common.common_repo_functions import has_primary_repo_remote, fetch_from_primary_repo, in_sync_with_primary
from edkrepo.common.common_repo_functions import update_hooks, combinations_in_manifest
from edkrepo.common.common_repo_functions import write_included_config, remove_included_config
from edkrepo.common.workspace_maintenance.git_config_maintenance import clean_git_globalconfig
from edkrepo.common.workspace_maintenance.git_config_maintenance import clean_git_globalconfig
from edkrepo.common.workspace_maintenance.workspace_maintenance import generate_name_for_obsolete_backup
from edkrepo.common.workspace_maintenance.manifest_repos_maintenance import pull_workspace_manifest_repo
from edkrepo.common.workspace_maintenance.manifest_repos_maintenance import find_source_manifest_repo
from edkrepo.common.workspace_maintenance.manifest_repos_maintenance import list_available_manifest_repos
from edkrepo.common.ui_functions import init_color_console
from edkrepo.config.config_factory import get_workspace_path, get_workspace_manifest, get_edkrepo_global_data_directory
from edkrepo.config.config_factory import get_workspace_manifest_file
from edkrepo.config.tool_config import SUBMODULE_CACHE_REPO_NAME
from edkrepo.config.tool_config import SUBMODULE_CACHE_REPO_NAME
from edkrepo_manifest_parser.edk_manifest import CiIndexXml, ManifestXml
from project_utils.submodule import deinit_submodules, maintain_submodules

Expand Down Expand Up @@ -105,7 +105,7 @@ def run_command(self, args, config):
if not args.update_local_manifest:
self.__check_for_new_manifest(args, config, initial_manifest, workspace_path, global_manifest_directory)
check_dirty_repos(initial_manifest, workspace_path)

# Determine if sparse checkout needs to be disabled for this operation
sparse_settings = initial_manifest.sparse_settings
sparse_enabled = sparse_checkout_enabled(workspace_path, initial_sources)
Expand All @@ -119,7 +119,7 @@ def run_command(self, args, config):
reset_sparse_checkout(workspace_path, initial_sources)

# Get the latest manifest if requested
if args.update_local_manifest: # NOTE: hyphens in arg name replaced with underscores due to argparse
if args.update_local_manifest: # NOTE: hyphens in arg name replaced with underscores due to argparse
self.__update_local_manifest(args, config, initial_manifest, workspace_path, global_manifest_directory)
manifest = get_workspace_manifest()
if args.update_local_manifest:
Expand Down Expand Up @@ -153,7 +153,7 @@ def run_command(self, args, config):
# Update submodule configuration
if not args.update_local_manifest: #Performance optimization, __update_local_manifest() will do this
self.__check_submodule_config(workspace_path, manifest, repo_sources_to_sync)
clean_git_globalconfig()
clean_git_globalconfig()
for repo_to_sync in repo_sources_to_sync:
local_repo_path = os.path.join(workspace_path, repo_to_sync.root)
# Update any hooks
Expand Down Expand Up @@ -215,11 +215,11 @@ def run_command(self, args, config):

# Initialize submodules
if not args.skip_submodule:
cache_path = None
cache_obj = get_repo_cache_obj(config)
if cache_obj is not None:
cache_path = cache_obj.get_cache_path(SUBMODULE_CACHE_REPO_NAME)
maintain_submodules(workspace_path, manifest, current_combo, args.verbose, cache_path)
cache_path = None
cache_obj = get_repo_cache_obj(config)
if cache_obj is not None:
cache_path = cache_obj.get_cache_path(SUBMODULE_CACHE_REPO_NAME)
maintain_submodules(workspace_path, manifest, current_combo, args.verbose, cache_path)

# Restore sparse checkout state
if sparse_enabled:
Expand Down Expand Up @@ -262,12 +262,6 @@ def __update_local_manifest(self, args, config, initial_manifest, workspace_path
write_included_config(new_manifest_to_check.remotes, new_manifest_to_check.submodule_alternate_remotes, local_manifest_dir)

self.__check_submodule_config(workspace_path, new_manifest_to_check, new_sources_for_current_combo)
new_manifest_remotes = {name:url for name, url in new_manifest_to_check.remotes}
#check for changes to remote urls
for remote_name in initial_manifest_remotes.keys():
if remote_name in new_manifest_remotes.keys():
if initial_manifest_remotes[remote_name] != new_manifest_remotes[remote_name]:
raise EdkrepoManifestChangedException(SYNC_URL_CHANGE.format(remote_name))

# Check that the repo sources lists are the same. If they are not the same and the override flag is not set, throw an exception.
if not args.override and set(initial_sources) != set(new_sources):
Expand All @@ -283,10 +277,13 @@ def __update_local_manifest(self, args, config, initial_manifest, workspace_path
if initial.root == new.root:
if initial.remote_name == new.remote_name:
if initial.remote_url == new.remote_url:
# If the source is unchanged between the old and the new manifest,
# add it to the common lists
common = True
initial_common.append(initial)
new_common.append(new)
break
# If the source is different between the old and the new manifest, add it to the uncommon list
if not common:
uncommon_sources.append(initial)
for new in new_sources:
Expand All @@ -297,6 +294,7 @@ def __update_local_manifest(self, args, config, initial_manifest, workspace_path
if new.remote_url == initial.remote_url:
common = True
break
# If the source is different between the old and the new manifest, add it to the uncommon list
if not common:
uncommon_sources.append(new)
uncommon_sources = set(uncommon_sources)
Expand All @@ -313,23 +311,34 @@ def __update_local_manifest(self, args, config, initial_manifest, workspace_path
if source_to_check.remote_url == source.remote_url:
found_source = True
break
# If the source that is different came from the old manifest, then it is now outdated and either needs
# to be deleted or moved to a archival location.
if found_source:
sources_to_remove.append(source)
else:
roots = [s.root for s in initial_sources]
roots = [s.root for s in new_sources]
# If there is a source in the new manifest that goes into the same folder name as a source in the
# old manifest, then we need to move that old folder to an archival location.
if source.root in roots:
#In theory, this should never happen, we should hit the SYNC_URL_CHANGE error
#this is here as a catch all
sources_to_move.append(source)
else:
sources_to_clone.append(source)
# If it doesn't exist at all in the new manifest, tell the user it is old and no longer used.
sources_to_remove.append(source)
else:
# If the source that is different came from the new manifest, then we need to clone that new
# Git repository.
sources_to_clone.append(source)
init_color_console(False)
# Move the obsolete Git repositories to archival locations.
for source in sources_to_move:
old_dir = os.path.join(workspace_path, source.root)
new_dir = generate_name_for_obsolete_backup(old_dir)
print(SYNC_SOURCE_MOVE_WARNING.format(source.root, new_dir))
new_dir = os.path.join(workspace_path, new_dir)
shutil.move(old_dir, new_dir)
try:
shutil.move(old_dir, new_dir)
except:
print(SYNC_MOVE_FAILED.format(initial_dir=source.root, new_dir=new_dir))
raise
# Tell the user about any Git repositories that are no longer used.
if len(sources_to_remove) > 0:
print(SYNC_REMOVE_WARNING)
for source in sources_to_remove:
Expand Down Expand Up @@ -454,4 +463,4 @@ def __check_submodule_config(self, workspace_path, manifest, repo_sources):
finally:
gitglobalconfig.release()


9 changes: 7 additions & 2 deletions edkrepo/common/humble.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
## @file
# humble.py
#
# Copyright (c) 2017 - 2020, Intel Corporation. All rights reserved.<BR>
# Copyright (c) 2017 - 2021, Intel Corporation. All rights reserved.<BR>
# SPDX-License-Identifier: BSD-2-Clause-Patent
#

Expand Down Expand Up @@ -57,7 +57,12 @@
FETCHING = 'Fetching latest code for {0} from {1} branch ...'
NO_SYNC_DETACHED_HEAD = 'No need to sync repo {0} since it is in detached HEAD state'
SYNC_MANIFEST_UPDATE = 'To update to the latest manifest please run edkrepo sync --update-local-manifest. {}'.format(Fore.RESET)
SYNC_REMOVE_LIST_END_FORMATTING = '{}'.format(Fore.RESET)
SYNC_REMOVE_LIST_END_FORMATTING = '{}'.format(Style.RESET_ALL)
SYNC_MOVE_FAILED = '''{}{}WARNING:{}{} Moving {{initial_dir}} to {{new_dir}} failed.
{}{}{}Most likely files from the original directory are open in an editor or IDE.
It is likely that {{new_dir}} contains your original code now.
Please close any open editors to reconcile this problem.
You may need to manually rename {{new_dir}} to {{initial_dir}} in some circumstances.\n{}'''.format(Style.BRIGHT, Fore.RED, Style.RESET_ALL, Fore.RED, Style.RESET_ALL, Style.BRIGHT, Fore.YELLOW, Style.RESET_ALL)

#error messages for clone_command.py
CLONE_EXIT = '\nExiting without performing clone operation.'
Expand Down