From 8338b786b05a9e2160a3020b6e59154f5dcd4813 Mon Sep 17 00:00:00 2001 From: Bill Sacks Date: Fri, 13 Jan 2023 17:25:55 -0700 Subject: [PATCH] Squashed 'manage_externals/' changes from a3b3a0373..82d3b247f MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 82d3b247f Merge pull request #178 from johnpaulalex/test_doc 3223f49ea Additional documentation of system tests - global variables, method descriptions. 45b7c01c3 Merge pull request #177 from jedwards4b/git_workflow ace90b2c2 try setting credentials this way f4d6aa933 try setting credentials this way 1d61a6944 use this to set git credentials 7f9d330e1 use this to set git credentials 5ac731b85 add tmate code 836847be7 get git workflow working dcd462d71 Merge pull request #176 from jedwards4b/add_github_testing 2d2479e9d Merge pull request #175 from johnpaulalex/fix 711a53fdf add github testing of prs and automatic tagging of main cfe0f888a fix typos 5665d6140 Fix broken checkout behavior introduced by PR #172. 27909e255 Merge pull request #173 from johnpaulalex/readall 00ad0440b Further tiny refactorings and docs of checkout API (no-op). Remove unused load_all param in _External.checkout(). Rename _External.checkout_externals() to checkout_subexternals(), to remove the ambiguity about whether the main external pointed to by the _External is itelf checked out (it is not) Clarify load_all documentation - it’s always recursive, but applies different criteria at each level. Rename variables in checkout.py (e.g. ext_description) to match the equivalent code in sourcetree.py. 2ea3d1a3a Merge pull request #172 from johnpaulalex/fixit 43bf8092c Merge pull request #171 from johnpaulalex/docstatus e6aa7d21e Merge pull request #170 from johnpaulalex/printdir adbd71557 On checkout, refresh locally installed optional packages regardless of whether -o is passed in. add074593 Comment tweaks, and fix 'ppath' typo 696527cb8 Document the format of various status dictionaries, and the various paths and path components within an _External. c677b9403 When processing an external, print out its path in addition to the base filename (to disambiguate all the externals.cfg's) 975d7fd5a Merge pull request #169 from johnpaulalex/docfix_branch 09709e36d Document _Externals.status(). The original comment was apparently copy-pasted from checkout(). 1d880e090 Merge pull request #167 from billsacks/fix_svn_on_windows 3510da848 Tweak a unit test to improve coverage eb7fc1368 Handle the possibility that the URL already ends with '/' 02ea87e3d Fix svn URLs on Windows b1c02ab54 Merge pull request #165 from gold2718/doc_fix 9f4be8c7b Add documentation about externals = None feature git-subtree-dir: manage_externals git-subtree-split: 82d3b247fc8fb3b70458bf2b4570ee6678754c4d --- .github/workflows/bumpversion.yml | 19 + .github/workflows/tests.yml | 30 ++ README.md | 5 + manic/checkout.py | 92 ++--- manic/externals_description.py | 19 +- manic/externals_status.py | 30 +- manic/repository_factory.py | 1 + manic/repository_svn.py | 9 +- manic/sourcetree.py | 275 ++++++++------ test/README.md | 38 +- test/test_sys_checkout.py | 591 ++++++++++++++++-------------- test/test_unit_repository_svn.py | 2 +- 12 files changed, 620 insertions(+), 491 deletions(-) create mode 100644 .github/workflows/bumpversion.yml create mode 100644 .github/workflows/tests.yml mode change 100644 => 100755 test/test_unit_repository_svn.py diff --git a/.github/workflows/bumpversion.yml b/.github/workflows/bumpversion.yml new file mode 100644 index 0000000000..f4dc9b7ca5 --- /dev/null +++ b/.github/workflows/bumpversion.yml @@ -0,0 +1,19 @@ +name: Bump version +on: + push: + branches: + - main +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Bump version and push tag + id: tag_version + uses: mathieudutour/github-tag-action@v5.5 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + create_annotated_tag: true + default_bump: patch + dry_run: false + tag_prefix: manic- diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000000..dd75b91b49 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,30 @@ +# This is a workflow to compile the cmeps source without cime +name: Test Manic + +# Controls when the action will run. Triggers the workflow on push or pull request +# events but only for the master branch +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +jobs: + test-manic: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Test Manic + run: | + pushd test + git config --global user.email "devnull@example.com" + git config --global user.name "GITHUB tester" + git config --global protocol.file.allow always + make utest + make stest + popd + + - name: Setup tmate session + if: ${{ failure() }} + uses: mxschmitt/action-tmate@v3 diff --git a/README.md b/README.md index c931c8e213..9475301b5d 100644 --- a/README.md +++ b/README.md @@ -201,6 +201,11 @@ The root of the source tree will be referred to as `${SRC_ROOT}` below. externals description file pointed 'useful_library/sub-xternals.cfg', Then the main 'externals' field in the top level repo should point to 'sub-externals.cfg'. + Note that by default, `checkout_externals` will clone an external's + submodules. As a special case, the entry, `externals = None`, will + prevent this behavior. For more control over which externals are + checked out, create an externals file (and see the `from_submodule` + configuration entry below). * from_submodule (True / False) : used to pull the repo_url, local_path, and hash properties for this external from the .gitmodules file in diff --git a/manic/checkout.py b/manic/checkout.py index 2223b1f0d8..ac30f3a5d2 100755 --- a/manic/checkout.py +++ b/manic/checkout.py @@ -227,6 +227,12 @@ def commandline_arguments(args=None): Now, %(prog)s will process Externals.cfg and also process Externals_LIBX.cfg as if it was a sub-external. + Note that by default, checkout_externals will clone an external's + submodules. As a special case, the entry, "externals = None", will + prevent this behavior. For more control over which externals are + checked out, create an externals file (and see the from_submodule + configuration entry below). + * from_submodule (True / False) : used to pull the repo_url, local_path, and hash properties for this external from the .gitmodules file in this repository. Note that the section name (the entry in square @@ -332,7 +338,34 @@ def commandline_arguments(args=None): options = parser.parse_args() return options - +def _dirty_local_repo_msg(program_name, config_file): + return """The external repositories labeled with 'M' above are not in a clean state. +The following are four options for how to proceed: +(1) Go into each external that is not in a clean state and issue either a 'git status' or + an 'svn status' command (depending on whether the external is managed by git or + svn). Either revert or commit your changes so that all externals are in a clean + state. (To revert changes in git, follow the instructions given when you run 'git + status'.) (Note, though, that it is okay to have untracked files in your working + directory.) Then rerun {program_name}. +(2) Alternatively, you do not have to rely on {program_name}. Instead, you can manually + update out-of-sync externals (labeled with 's' above) as described in the + configuration file {config_file}. (For example, run 'git fetch' and 'git checkout' + commands to checkout the appropriate tags for each external, as given in + {config_file}.) +(3) You can also use {program_name} to manage most, but not all externals: You can specify + one or more externals to ignore using the '-x' or '--exclude' argument to + {program_name}. Excluding externals labeled with 'M' will allow {program_name} to + update the other, non-excluded externals. +(4) As a last resort, if you are confident that there is no work that needs to be saved + from a given external, you can remove that external (via "rm -rf [directory]") and + then rerun the {program_name} tool. This option is mainly useful as a workaround for + issues with this tool (such as https://github.com/ESMCI/manage_externals/issues/157). +The external repositories labeled with '?' above are not under version +control using the expected protocol. If you are sure you want to switch +protocols, and you don't have any work you need to save from this +directory, then run "rm -rf [directory]" before rerunning the +{program_name} tool. +""".format(program_name=program_name, config_file=config_file) # --------------------------------------------------------------------- # # main @@ -363,19 +396,25 @@ def main(args): load_all = True root_dir = os.path.abspath(os.getcwd()) - external_data = read_externals_description_file(root_dir, args.externals) - external = create_externals_description( - external_data, components=args.components, exclude=args.exclude) + model_data = read_externals_description_file(root_dir, args.externals) + ext_description = create_externals_description( + model_data, components=args.components, exclude=args.exclude) for comp in args.components: - if comp not in external.keys(): + if comp not in ext_description.keys(): + # Note we can't print out the list of found externals because + # they were filtered in create_externals_description above. fatal_error( "No component {} found in {}".format( comp, args.externals)) - source_tree = SourceTree(root_dir, external, svn_ignore_ancestry=args.svn_ignore_ancestry) - printlog('Checking status of externals: ', end='') - tree_status = source_tree.status() + source_tree = SourceTree(root_dir, ext_description, svn_ignore_ancestry=args.svn_ignore_ancestry) + if args.components: + components_str = 'specified components' + else: + components_str = 'required & optional components' + printlog('Checking local status of ' + components_str + ': ', end='') + tree_status = source_tree.status(print_progress=True) printlog('') if args.status: @@ -390,43 +429,8 @@ def main(args): for comp in sorted(tree_status): tree_status[comp].log_status_message(args.verbose) # exit gracefully - msg = """The external repositories labeled with 'M' above are not in a clean state. - -The following are four options for how to proceed: - -(1) Go into each external that is not in a clean state and issue either a 'git status' or - an 'svn status' command (depending on whether the external is managed by git or - svn). Either revert or commit your changes so that all externals are in a clean - state. (To revert changes in git, follow the instructions given when you run 'git - status'.) (Note, though, that it is okay to have untracked files in your working - directory.) Then rerun {program_name}. - -(2) Alternatively, you do not have to rely on {program_name}. Instead, you can manually - update out-of-sync externals (labeled with 's' above) as described in the - configuration file {config_file}. (For example, run 'git fetch' and 'git checkout' - commands to checkout the appropriate tags for each external, as given in - {config_file}.) - -(3) You can also use {program_name} to manage most, but not all externals: You can specify - one or more externals to ignore using the '-x' or '--exclude' argument to - {program_name}. Excluding externals labeled with 'M' will allow {program_name} to - update the other, non-excluded externals. - -(4) As a last resort, if you are confident that there is no work that needs to be saved - from a given external, you can remove that external (via "rm -rf [directory]") and - then rerun the {program_name} tool. This option is mainly useful as a workaround for - issues with this tool (such as https://github.com/ESMCI/manage_externals/issues/157). - - -The external repositories labeled with '?' above are not under version -control using the expected protocol. If you are sure you want to switch -protocols, and you don't have any work you need to save from this -directory, then run "rm -rf [directory]" before rerunning the -{program_name} tool. -""".format(program_name=program_name, config_file=args.externals) - printlog('-' * 70) - printlog(msg) + printlog(_dirty_local_repo_msg(program_name, args.externals)) printlog('-' * 70) else: if not args.components: diff --git a/manic/externals_description.py b/manic/externals_description.py index 6a54935935..f5615b6730 100644 --- a/manic/externals_description.py +++ b/manic/externals_description.py @@ -71,7 +71,8 @@ def read_externals_description_file(root_dir, file_name): root_dir = os.path.abspath(root_dir) msg = 'In directory : {0}'.format(root_dir) logging.info(msg) - printlog('Processing externals description file : {0}'.format(file_name)) + printlog('Processing externals description file : {0} ({1})'.format(file_name, + root_dir)) file_path = os.path.join(root_dir, file_name) if not os.path.exists(file_name): @@ -281,6 +282,10 @@ def read_gitmodules_file(root_dir, file_name): def create_externals_description( model_data, model_format='cfg', components=None, exclude=None, parent_repo=None): """Create the a externals description object from the provided data + + components: list of component names to include, None to include all. If a + name isn't found, it is silently omitted from the return value. + exclude: list of component names to skip. """ externals_description = None if model_format == 'dict': @@ -357,8 +362,9 @@ class ExternalsDescription(dict): input value. """ - # keywords defining the interface into the externals description data - EXTERNALS = 'externals' + # keywords defining the interface into the externals description data; these + # are brought together by the schema below. + EXTERNALS = 'externals' # path to externals file. BRANCH = 'branch' SUBMODULE = 'from_submodule' HASH = 'hash' @@ -384,6 +390,8 @@ class ExternalsDescription(dict): _V1_BRANCH = 'BRANCH' _V1_REQ_SOURCE = 'REQ_SOURCE' + # Dictionary keys are component names. The corresponding values are laid out + # according to this schema. _source_schema = {REQUIRED: True, PATH: 'string', EXTERNALS: 'string', @@ -760,6 +768,8 @@ def __init__(self, model_data, components=None, exclude=None, parent_repo=None): """Convert the config data into a standardized dict that can be used to construct the source objects + components: list of component names to include, None to include all. + exclude: list of component names to skip. """ ExternalsDescription.__init__(self, parent_repo=parent_repo) self._schema_major = 1 @@ -783,6 +793,9 @@ def _remove_metadata(model_data): def _parse_cfg(self, cfg_data, components=None, exclude=None): """Parse a config_parser object into a externals description. + + components: list of component names to include, None to include all. + exclude: list of component names to skip. """ def list_to_dict(input_list, convert_to_lower_case=True): """Convert a list of key-value pairs into a dictionary. diff --git a/manic/externals_status.py b/manic/externals_status.py index d3d238f289..4900e41254 100644 --- a/manic/externals_status.py +++ b/manic/externals_status.py @@ -29,16 +29,16 @@ class ExternalStatus(object): transactions (e.g. add, remove, rename, untracked files). """ - DEFAULT = '-' + # sync_state and clean_state can be one of the following: + DEFAULT = '-' # aka not set yet. UNKNOWN = '?' EMPTY = 'e' - MODEL_MODIFIED = 's' # a.k.a. out-of-sync - DIRTY = 'M' - - STATUS_OK = ' ' + MODEL_MODIFIED = 's' # repo version != externals (sync_state only) + DIRTY = 'M' # repo is dirty (clean_state only) + STATUS_OK = ' ' # repo is clean/matches externals. STATUS_ERROR = '!' - # source types + # source_type can be one of the following: OPTIONAL = 'o' STANDALONE = 's' MANAGED = ' ' @@ -55,19 +55,21 @@ def __init__(self): def log_status_message(self, verbosity): """Write status message to the screen and log file """ - self._default_status_message() + printlog(self._default_status_message()) if verbosity >= VERBOSITY_VERBOSE: - self._verbose_status_message() + printlog(self._verbose_status_message()) if verbosity >= VERBOSITY_DUMP: - self._dump_status_message() + printlog(self._dump_status_message()) + + def __repr__(self): + return self._default_status_message() def _default_status_message(self): """Return the default terse status message string """ - msg = '{sync}{clean}{src_type} {path}'.format( + return '{sync}{clean}{src_type} {path}'.format( sync=self.sync_state, clean=self.clean_state, src_type=self.source_type, path=self.path) - printlog(msg) def _verbose_status_message(self): """Return the verbose status message string @@ -82,14 +84,12 @@ def _verbose_status_message(self): if self.sync_state != self.STATUS_OK: sync_str = '{current} --> {expected}'.format( current=self.current_version, expected=self.expected_version) - msg = ' {clean}, {sync}'.format(clean=clean_str, sync=sync_str) - printlog(msg) + return ' {clean}, {sync}'.format(clean=clean_str, sync=sync_str) def _dump_status_message(self): """Return the dump status message string """ - msg = indent_string(self.status_output, 12) - printlog(msg) + return indent_string(self.status_output, 12) def safe_to_update(self): """Report if it is safe to update a repository. Safe is defined as: diff --git a/manic/repository_factory.py b/manic/repository_factory.py index 80a92a9d8a..18c73ffc4b 100644 --- a/manic/repository_factory.py +++ b/manic/repository_factory.py @@ -15,6 +15,7 @@ def create_repository(component_name, repo_info, svn_ignore_ancestry=False): """Determine what type of repository we have, i.e. git or svn, and create the appropriate object. + Can return None (e.g. if protocol is 'externals_only'). """ protocol = repo_info[ExternalsDescription.PROTOCOL].lower() if protocol == 'git': diff --git a/manic/repository_svn.py b/manic/repository_svn.py index 408ed84676..922855d34e 100644 --- a/manic/repository_svn.py +++ b/manic/repository_svn.py @@ -43,10 +43,15 @@ def __init__(self, component_name, repo, ignore_ancestry=False): """ Repository.__init__(self, component_name, repo) self._ignore_ancestry = ignore_ancestry + if self._url.endswith('/'): + # there is already a '/' separator in the URL; no need to add another + url_sep = '' + else: + url_sep = '/' if self._branch: - self._url = os.path.join(self._url, self._branch) + self._url = self._url + url_sep + self._branch elif self._tag: - self._url = os.path.join(self._url, self._tag) + self._url = self._url + url_sep + self._tag else: msg = "DEV_ERROR in svn repository. Shouldn't be here!" fatal_error(msg) diff --git a/manic/sourcetree.py b/manic/sourcetree.py index 54de763c30..218b5febb4 100644 --- a/manic/sourcetree.py +++ b/manic/sourcetree.py @@ -19,7 +19,7 @@ class _External(object): """ - _External represents an external object inside a SourceTree + A single component hosted in an external repository (and any children). """ # pylint: disable=R0902 @@ -41,31 +41,40 @@ def __init__(self, root_dir, name, ext_description, svn_ignore_ancestry): """ self._name = name - self._repo = None - self._externals = EMPTY_STR + self._repo = None # Repository object. + + # Subcomponent externals file and data object, if any. + self._externals_path = EMPTY_STR # Can also be "none" self._externals_sourcetree = None - self._stat = ExternalStatus() + + self._stat = None # Populated in status() self._sparse = None # Parse the sub-elements - # _path : local path relative to the containing source tree + # _local_path : local path relative to the containing source tree, e.g. + # "components/mom" self._local_path = ext_description[ExternalsDescription.PATH] - # _repo_dir : full repository directory + # _repo_dir_path : full repository directory, e.g. + # "/components/mom" repo_dir = os.path.join(root_dir, self._local_path) self._repo_dir_path = os.path.abspath(repo_dir) - # _base_dir : base directory *containing* the repository + # _base_dir_path : base directory *containing* the repository, e.g. + # "/components" self._base_dir_path = os.path.dirname(self._repo_dir_path) - # repo_dir_name : base_dir_path + repo_dir_name = rep_dir_path + # _repo_dir_name : base_dir_path + repo_dir_name = rep_dir_path + # e.g., "mom" self._repo_dir_name = os.path.basename(self._repo_dir_path) assert(os.path.join(self._base_dir_path, self._repo_dir_name) == self._repo_dir_path) self._required = ext_description[ExternalsDescription.REQUIRED] - self._externals = ext_description[ExternalsDescription.EXTERNALS] + + # Does this component have subcomponents aka an externals config? + self._externals_path = ext_description[ExternalsDescription.EXTERNALS] # Treat a .gitmodules file as a backup externals config - if not self._externals: + if not self._externals_path: if GitRepository.has_submodules(self._repo_dir_path): - self._externals = ExternalsDescription.GIT_SUBMODULES_FILENAME + self._externals_path = ExternalsDescription.GIT_SUBMODULES_FILENAME repo = create_repository( name, ext_description[ExternalsDescription.REPO], @@ -73,7 +82,8 @@ def __init__(self, root_dir, name, ext_description, svn_ignore_ancestry): if repo: self._repo = repo - if self._externals and (self._externals.lower() != 'none'): + # Recurse into subcomponents, if any. + if self._externals_path and (self._externals_path.lower() != 'none'): self._create_externals_sourcetree() def get_name(self): @@ -88,81 +98,88 @@ def get_local_path(self): """ return self._local_path - def status(self): - """ - If the repo destination directory exists, ensure it is correct (from - correct URL, correct branch or tag), and possibly update the external. - If the repo destination directory does not exist, checkout the correce - branch or tag. - If load_all is True, also load all of the the externals sub-externals. + def status(self, force=False, print_progress=False): """ + Returns status of this component and all subcomponents. - self._stat.path = self.get_local_path() - if not self._required: - self._stat.source_type = ExternalStatus.OPTIONAL - elif self._local_path == LOCAL_PATH_INDICATOR: - # LOCAL_PATH_INDICATOR, '.' paths, are standalone - # component directories that are not managed by - # checkout_externals. - self._stat.source_type = ExternalStatus.STANDALONE - else: - # managed by checkout_externals - self._stat.source_type = ExternalStatus.MANAGED + Returns a dict mapping our local path (not component name!) to an + ExternalStatus dict. Any subcomponents will have their own top-level + path keys. Note the return value includes entries for this and all + subcomponents regardless of whether they are locally installed or not. - ext_stats = {} + Side-effect: If self._stat is empty or force is True, calculates _stat. + """ + calc_stat = force or not self._stat + + if calc_stat: + self._stat = ExternalStatus() + self._stat.path = self.get_local_path() + if not self._required: + self._stat.source_type = ExternalStatus.OPTIONAL + elif self._local_path == LOCAL_PATH_INDICATOR: + # LOCAL_PATH_INDICATOR, '.' paths, are standalone + # component directories that are not managed by + # checkout_subexternals. + self._stat.source_type = ExternalStatus.STANDALONE + else: + # managed by checkout_subexternals + self._stat.source_type = ExternalStatus.MANAGED + subcomponent_stats = {} if not os.path.exists(self._repo_dir_path): - self._stat.sync_state = ExternalStatus.EMPTY - msg = ('status check: repository directory for "{0}" does not ' - 'exist.'.format(self._name)) - logging.info(msg) - self._stat.current_version = 'not checked out' - # NOTE(bja, 2018-01) directory doesn't exist, so we cannot - # use repo to determine the expected version. We just take - # a best-guess based on the assumption that only tag or - # branch should be set, but not both. - if not self._repo: - self._stat.expected_version = 'unknown' - else: - self._stat.expected_version = self._repo.tag() + self._repo.branch() + if calc_stat: + # No local repository. + self._stat.sync_state = ExternalStatus.EMPTY + msg = ('status check: repository directory for "{0}" does not ' + 'exist.'.format(self._name)) + logging.info(msg) + self._stat.current_version = 'not checked out' + # NOTE(bja, 2018-01) directory doesn't exist, so we cannot + # use repo to determine the expected version. We just take + # a best-guess based on the assumption that only tag or + # branch should be set, but not both. + if not self._repo: + self._stat.expected_version = 'unknown' + else: + self._stat.expected_version = self._repo.tag() + self._repo.branch() else: - if self._repo: + # Merge local repository state (e.g. clean/dirty) into self._stat. + if calc_stat and self._repo: self._repo.status(self._stat, self._repo_dir_path) - if self._externals and self._externals_sourcetree: - # we expect externals and they exist + # Status of subcomponents, if any. + if self._externals_path and self._externals_sourcetree: cwd = os.getcwd() - # SourceTree expects to be called from the correct + # SourceTree.status() expects to be called from the correct # root directory. os.chdir(self._repo_dir_path) - ext_stats = self._externals_sourcetree.status(self._local_path) + subcomponent_stats = self._externals_sourcetree.status(self._local_path, force=force, print_progress=print_progress) os.chdir(cwd) + # Merge our status + subcomponent statuses into one return dict keyed + # by component path. all_stats = {} # don't add the root component because we don't manage it # and can't provide useful info about it. if self._local_path != LOCAL_PATH_INDICATOR: - # store the stats under tha local_path, not comp name so + # store the stats under the local_path, not comp name so # it will be sorted correctly all_stats[self._stat.path] = self._stat - if ext_stats: - all_stats.update(ext_stats) + if subcomponent_stats: + all_stats.update(subcomponent_stats) return all_stats - def checkout(self, verbosity, load_all): + def checkout(self, verbosity): """ If the repo destination directory exists, ensure it is correct (from correct URL, correct branch or tag), and possibly update the external. If the repo destination directory does not exist, checkout the correct branch or tag. - If load_all is True, also load all of the the externals sub-externals. + Does not check out sub-externals, see checkout_subexternals(). """ - if load_all: - pass # Make sure we are in correct location - if not os.path.exists(self._repo_dir_path): # repository directory doesn't exist. Need to check it # out, and for that we need the base_dir_path to exist @@ -174,6 +191,10 @@ def checkout(self, verbosity, load_all): self._base_dir_path) fatal_error(msg) + if not self._stat: + self.status() + assert self._stat + if self._stat.source_type != ExternalStatus.STANDALONE: if verbosity >= VERBOSITY_VERBOSE: # NOTE(bja, 2018-01) probably do not want to pass @@ -194,8 +215,10 @@ def checkout(self, verbosity, load_all): self._repo.checkout(self._base_dir_path, self._repo_dir_name, checkout_verbosity, self.clone_recursive()) - def checkout_externals(self, verbosity, load_all): - """Checkout the sub-externals for this object + def checkout_subexternals(self, verbosity, load_all): + """Recursively checkout the sub-externals for this component, if any. + + See load_all documentation in SourceTree.checkout(). """ if self.load_externals(): if self._externals_sourcetree: @@ -210,25 +233,26 @@ def checkout_externals(self, verbosity, load_all): self._externals_sourcetree.checkout(verbosity, load_all) def load_externals(self): - 'Return True iff an externals file should be loaded' + 'Return True iff an externals file exists (and therefore should be loaded)' load_ex = False if os.path.exists(self._repo_dir_path): - if self._externals: - if self._externals.lower() != 'none': + if self._externals_path: + if self._externals_path.lower() != 'none': load_ex = os.path.exists(os.path.join(self._repo_dir_path, - self._externals)) + self._externals_path)) return load_ex def clone_recursive(self): 'Return True iff any .gitmodules files should be processed' - # Try recursive unless there is an externals entry - recursive = not self._externals + # Try recursive .gitmodules unless there is an externals entry + recursive = not self._externals_path return recursive def _create_externals_sourcetree(self): """ + Note this only creates an object, it doesn't write to the file system. """ if not os.path.exists(self._repo_dir_path): # NOTE(bja, 2017-10) repository has not been checked out @@ -239,29 +263,31 @@ def _create_externals_sourcetree(self): cwd = os.getcwd() os.chdir(self._repo_dir_path) - if self._externals.lower() == 'none': + if self._externals_path.lower() == 'none': msg = ('Internal: Attempt to create source tree for ' 'externals = none in {}'.format(self._repo_dir_path)) fatal_error(msg) - if not os.path.exists(self._externals): + if not os.path.exists(self._externals_path): if GitRepository.has_submodules(): - self._externals = ExternalsDescription.GIT_SUBMODULES_FILENAME + self._externals_path = ExternalsDescription.GIT_SUBMODULES_FILENAME - if not os.path.exists(self._externals): + if not os.path.exists(self._externals_path): # NOTE(bja, 2017-10) this check is redundent with the one # in read_externals_description_file! msg = ('External externals description file "{0}" ' 'does not exist! In directory: {1}'.format( - self._externals, self._repo_dir_path)) + self._externals_path, self._repo_dir_path)) fatal_error(msg) externals_root = self._repo_dir_path + # model_data is a dict-like object which mirrors the file format. model_data = read_externals_description_file(externals_root, - self._externals) - externals = create_externals_description(model_data, - parent_repo=self._repo) - self._externals_sourcetree = SourceTree(externals_root, externals) + self._externals_path) + # ext_description is another dict-like object (see ExternalsDescription) + ext_description = create_externals_description(model_data, + parent_repo=self._repo) + self._externals_sourcetree = SourceTree(externals_root, ext_description) os.chdir(cwd) class SourceTree(object): @@ -269,45 +295,48 @@ class SourceTree(object): SourceTree represents a group of managed externals """ - def __init__(self, root_dir, model, svn_ignore_ancestry=False): + def __init__(self, root_dir, ext_description, svn_ignore_ancestry=False): """ - Build a SourceTree object from a model description + Build a SourceTree object from an ExternalDescription. """ self._root_dir = os.path.abspath(root_dir) - self._all_components = {} + self._all_components = {} # component_name -> _External self._required_compnames = [] - for comp in model: - src = _External(self._root_dir, comp, model[comp], svn_ignore_ancestry) + for comp in ext_description: + src = _External(self._root_dir, comp, ext_description[comp], + svn_ignore_ancestry) self._all_components[comp] = src - if model[comp][ExternalsDescription.REQUIRED]: + if ext_description[comp][ExternalsDescription.REQUIRED]: self._required_compnames.append(comp) - def status(self, relative_path_base=LOCAL_PATH_INDICATOR): - """Report the status components - - FIXME(bja, 2017-10) what do we do about situations where the - user checked out the optional components, but didn't add - optional for running status? What do we do where the user - didn't add optional to the checkout but did add it to the - status. -- For now, we run status on all components, and try - to do the right thing based on the results.... - - """ + def status(self, relative_path_base=LOCAL_PATH_INDICATOR, + force=False, print_progress=False): + """Return a dictionary of local path->ExternalStatus. + + Notes about the returned dictionary: + * It is keyed by local path (e.g. 'components/mom'), not by + component name (e.g. 'mom'). + * It contains top-level keys for all traversed components, whether + discovered by recursion or top-level. + * It contains entries for all components regardless of whether they + are locally installed or not, or required or optional. +x """ load_comps = self._all_components.keys() - summary = {} + summary = {} # Holds merged statuses from all components. for comp in load_comps: - printlog('{0}, '.format(comp), end='') - stat = self._all_components[comp].status() + if print_progress: + printlog('{0}, '.format(comp), end='') + stat = self._all_components[comp].status(force=force, + print_progress=print_progress) + + # Returned status dictionary is keyed by local path; prepend + # relative_path_base if not already there. stat_final = {} for name in stat.keys(): - # check if we need to append the relative_path_base to - # the path so it will be sorted in the correct order. if stat[name].path.startswith(relative_path_base): - # use as is, without any changes to path stat_final[name] = stat[name] else: - # append relative_path_base to path and store under key = updated path modified_path = os.path.join(relative_path_base, stat[name].path) stat_final[modified_path] = stat[name] @@ -316,30 +345,48 @@ def status(self, relative_path_base=LOCAL_PATH_INDICATOR): return summary + def _find_installed_optional_components(self): + """Returns a list of installed optional component names, if any.""" + installed_comps = [] + for comp_name, ext in self._all_components.items(): + if comp_name in self._required_compnames: + continue + # Note that in practice we expect this status to be cached. + path_to_stat = ext.status() + if any(stat.sync_state != ExternalStatus.EMPTY + for stat in path_to_stat.values()): + installed_comps.append(comp_name) + return installed_comps + def checkout(self, verbosity, load_all, load_comp=None): """ - Checkout or update indicated components into the the configured - subdirs. + Checkout or update indicated components into the configured subdirs. - If load_all is True, recursively checkout all externals. - If load_all is False, load_comp is an optional set of components to load. - If load_all is True and load_comp is None, only load the required externals. + If load_all is True, checkout all externals (required + optional), recursively. + If load_all is False and load_comp is set, checkout load_comp (and any required subexternals, plus any optional subexternals that are already checked out, recursively) + If load_all is False and load_comp is None, checkout all required externals, plus any optionals that are already checked out, recursively. """ - if verbosity >= VERBOSITY_VERBOSE: - printlog('Checking out externals: ') - else: - printlog('Checking out externals: ', end='') - if load_all: tmp_comps = self._all_components.keys() elif load_comp is not None: tmp_comps = [load_comp] else: - tmp_comps = self._required_compnames + local_optional_compnames = self._find_installed_optional_components() + tmp_comps = self._required_compnames + local_optional_compnames + if local_optional_compnames: + printlog('Found locally installed optional components: ' + + ', '.join(local_optional_compnames)) + + if verbosity >= VERBOSITY_VERBOSE: + printlog('Checking out externals: ') + else: + printlog('Checking out externals: ', end='') + # Sort by path so that if paths are nested the # parent repo is checked out first. load_comps = sorted(tmp_comps, key=lambda comp: self._all_components[comp].get_local_path()) - # checkout the primary externals + + # checkout. for comp in load_comps: if verbosity < VERBOSITY_VERBOSE: printlog('{0}, '.format(comp), end='') @@ -347,7 +394,9 @@ def checkout(self, verbosity, load_all, load_comp=None): # verbose output handled by the _External object, just # output a newline printlog(EMPTY_STR) - self._all_components[comp].checkout(verbosity, load_all) - # now give each external an opportunitity to checkout it's externals. - self._all_components[comp].checkout_externals(verbosity, load_all) + # Does not recurse. + self._all_components[comp].checkout(verbosity) + # Recursively check out subexternals, if any. + self._all_components[comp].checkout_subexternals(verbosity, + load_all) printlog('') diff --git a/test/README.md b/test/README.md index 938a900eec..1e8f2eaa77 100644 --- a/test/README.md +++ b/test/README.md @@ -1,45 +1,25 @@ # Testing for checkout_externals -NOTE: Python2 is the supported runtime environment. Python3 compatibility is -in progress, complicated by the different proposed input methods -(yaml, xml, cfg/ini, json) and their different handling of strings -(unicode vs byte) in python2. Full python3 compatibility will be -possible once the number of possible input formats has been narrowed. - -## Setup development environment - -Development environments should be setup for python2 and python3: +## Unit tests ```SH cd checkout_externals/test - make python=python2 env - make python=python3 env + make utest ``` -## Unit tests - -Tests should be run for both python2 and python3. It is recommended -that you have seperate terminal windows open python2 and python3 -testing to avoid errors activating and deactivating environments. +## System tests ```SH cd checkout_externals/test - . env_python2/bin/activate - make utest - deactivate + make stest ``` +Example to run a single test: ```SH - cd checkout_externals/test - . env_python2/bin/activate - make utest - deactivate + cd checkout_externals + python -m unittest test.test_sys_checkout.TestSysCheckout.test_container_simple_required ``` -## System tests - -Not yet implemented. - ## Static analysis checkout_externals is difficult to test thoroughly because it relies @@ -51,9 +31,7 @@ regularly for automatic code formatting and linting. ```SH cd checkout_externals/test - . env_python2/bin/activate make lint - deactivate ``` The canonical formatting for the code is whatever autopep8 @@ -68,10 +46,8 @@ coverage, run the code coverage tool: ```SH cd checkout_externals/test - . env_python2/bin/activate make coverage open -a Firefox.app htmlcov/index.html - deactivate ``` diff --git a/test/test_sys_checkout.py b/test/test_sys_checkout.py index b13e96a17d..27b649bea4 100644 --- a/test/test_sys_checkout.py +++ b/test/test_sys_checkout.py @@ -66,27 +66,41 @@ # # --------------------------------------------------------------------- -# environment variable names -MANIC_TEST_BARE_REPO_ROOT = 'MANIC_TEST_BARE_REPO_ROOT' -MANIC_TEST_TMP_REPO_ROOT = 'MANIC_TEST_TMP_REPO_ROOT' -# directory names -TMP_REPO_DIR_NAME = 'tmp' -BARE_REPO_ROOT_NAME = 'repos' +# Module-wide root directory for all the per-test subdirs we'll create on +# the fly. +MANIC_TEST_TMP_REPO_ROOT = 'MANIC_TEST_TMP_REPO_ROOT' # env var name +TMP_REPO_DIR_NAME = 'tmp' # subdir under $CWD + +# We clone these checked-in repositories on a per-test basis. The +# 'bare repo root' is the test/repos/ subdir that holds them all. +MANIC_TEST_BARE_REPO_ROOT = 'MANIC_TEST_BARE_REPO_ROOT' # env var name +BARE_REPO_ROOT_NAME = 'repos' # subdir name + +# Subdirs under bare repo root, each holding a repository. CONTAINER_REPO_NAME = 'container.git' MIXED_REPO_NAME = 'mixed-cont-ext.git' SIMPLE_REPO_NAME = 'simple-ext.git' SIMPLE_FORK_NAME = 'simple-ext-fork.git' -SIMPLE_LOCAL_ONLY_NAME = '.' -ERROR_REPO_NAME = 'error' + + +# In each of the test repos, all externals live under this subdir. EXTERNALS_NAME = 'externals' -SUB_EXTERNALS_PATH = 'src' +SUB_EXTERNALS_PATH = 'src' # For mixed test repos, + +# For testing behavior with '.' instead of an explicit paths. +SIMPLE_LOCAL_ONLY_NAME = '.' + +# Filenames CFG_NAME = 'externals.cfg' CFG_SUB_NAME = 'sub-externals.cfg' -README_NAME = 'readme.txt' +README_NAME = 'readme.txt' # Arbitrary file to check in. + +# Arbirary test branch name REMOTE_BRANCH_FEATURE2 = 'feature2' -NESTED_NAME = ['./fred', './fred/wilma', './fred/wilma/barney'] +# Section names in container_nested_* test config. +NESTED_NAME = ['./fred', './fred/wilma', './fred/wilma/barney'] SVN_TEST_REPO = 'https://github.com/escomp/cesm' @@ -110,16 +124,18 @@ def setUpModule(): # pylint: disable=C0103 # create clean dir for this run os.mkdir(repo_root) # set into the environment so var will be expanded in externals - # filess when executables are run + # files when executables are run os.environ[MANIC_TEST_TMP_REPO_ROOT] = repo_root class GenerateExternalsDescriptionCfgV1(object): - """Class to provide building blocks to create - ExternalsDescriptionCfgV1 files. + """Building blocks to create ExternalsDescriptionCfgV1 files. - Includes predefined files used in tests. + Basic usage: create_config() multiple create_*(), then write_config(). + Optionally after that: write_with_*(). + Includes methods (like container_*()) to create predefined configs for the + checked-in repositories in test/repos/. """ def __init__(self): @@ -255,7 +271,7 @@ def mixed_simple_sub(self, dest_dir): self.write_config(dest_dir, filename=CFG_SUB_NAME) def write_config(self, dest_dir, filename=CFG_NAME): - """Write the configuration file to disk + """Write self._config to disk """ dest_path = os.path.join(dest_dir, filename) @@ -277,14 +293,17 @@ def create_metadata(self): self._config.set(DESCRIPTION_SECTION, VERSION_ITEM, self._schema_version) - def create_section(self, repo_type, name, tag='', branch='', + def create_section(self, repo_path, name, tag='', branch='', ref_hash='', required=True, path=EXTERNALS_NAME, - externals='', repo_path=None, from_submodule=False, + externals='', repo_path_abs=None, from_submodule=False, sparse='', nested=False): # pylint: disable=too-many-branches - """Create a config section with autofilling some items and handling - optional items. + """Create a config ExternalsDescription section with the given name. + + Autofills some items and handles some optional items. + repo_path_abs overrides repo_path (which is relative to the bare repo) + path is a subdir under repo_path. """ # pylint: disable=R0913 self._config.add_section(name) @@ -300,7 +319,7 @@ def create_section(self, repo_type, name, tag='', branch='', # from_submodules is incompatible with some other options, turn them off if (from_submodule and - ((repo_path is not None) or tag or ref_hash or branch)): + ((repo_path_abs is not None) or tag or ref_hash or branch)): printlog('create_section: "from_submodule" is incompatible with ' '"repo_url", "tag", "hash", and "branch" options;\n' 'Ignoring those options for {}'.format(name)) @@ -309,10 +328,10 @@ def create_section(self, repo_type, name, tag='', branch='', ref_hash = '' branch = '' - if repo_path is not None: - repo_url = repo_path + if repo_path_abs is not None: + repo_url = repo_path_abs else: - repo_url = os.path.join('${MANIC_TEST_BARE_REPO_ROOT}', repo_type) + repo_url = os.path.join('${MANIC_TEST_BARE_REPO_ROOT}', repo_path) if not from_submodule: self._config.set(name, ExternalsDescription.REPO_URL, repo_url) @@ -381,12 +400,11 @@ def create_svn_external(self, name, tag='', branch=''): @staticmethod def create_branch(dest_dir, repo_name, branch, with_commit=False): - """Update a repository branch, and potentially the remote. + """Create branch and optionally (with_commit) add a single commit. """ # pylint: disable=R0913 cwd = os.getcwd() - repo_root = os.path.join(dest_dir, EXTERNALS_NAME) - repo_root = os.path.join(repo_root, repo_name) + repo_root = os.path.join(dest_dir, EXTERNALS_NAME, repo_name) os.chdir(repo_root) cmd = ['git', 'checkout', '-b', branch, ] execute_subprocess(cmd) @@ -407,10 +425,11 @@ def create_commit(dest_dir, repo_name, local_tracking_branch=None): This is used to test sync state changes from local commits on detached heads and tracking branches. + NOTE(jpalex, 2023-01) This has nothing to do with self._config, + should probably live elsewhere. """ cwd = os.getcwd() - repo_root = os.path.join(dest_dir, EXTERNALS_NAME) - repo_root = os.path.join(repo_root, repo_name) + repo_root = os.path.join(dest_dir, EXTERNALS_NAME, repo_name) os.chdir(repo_root) if local_tracking_branch: cmd = ['git', 'checkout', '-b', local_tracking_branch, ] @@ -425,19 +444,21 @@ def create_commit(dest_dir, repo_name, local_tracking_branch=None): execute_subprocess(cmd) os.chdir(cwd) - def update_branch(self, dest_dir, name, branch, repo_type=None, - filename=CFG_NAME): - """Update a repository branch, and potentially the remote. + def write_with_git_branch(self, dest_dir, name, branch, repo_path=None, + filename=CFG_NAME): + """Update fields in our config and write it to disk. + + name is the key of the ExternalsDescription in self._config to update. """ # pylint: disable=R0913 self._config.set(name, ExternalsDescription.BRANCH, branch) - if repo_type: - if repo_type == SIMPLE_LOCAL_ONLY_NAME: + if repo_path: + if repo_path == SIMPLE_LOCAL_ONLY_NAME: repo_url = SIMPLE_LOCAL_ONLY_NAME else: repo_url = os.path.join('${MANIC_TEST_BARE_REPO_ROOT}', - repo_type) + repo_path) self._config.set(name, ExternalsDescription.REPO_URL, repo_url) try: @@ -448,7 +469,7 @@ def update_branch(self, dest_dir, name, branch, repo_type=None, self.write_config(dest_dir, filename) - def update_svn_branch(self, dest_dir, name, branch, filename=CFG_NAME): + def write_with_svn_branch(self, dest_dir, name, branch, filename=CFG_NAME): """Update a repository branch, and potentially the remote. """ # pylint: disable=R0913 @@ -462,7 +483,7 @@ def update_svn_branch(self, dest_dir, name, branch, filename=CFG_NAME): self.write_config(dest_dir, filename) - def update_tag(self, dest_dir, name, tag, repo_type=None, + def write_with_tag(self, dest_dir, name, tag, repo_path=None, filename=CFG_NAME, remove_branch=True): """Update a repository tag, and potentially the remote @@ -474,8 +495,8 @@ def update_tag(self, dest_dir, name, tag, repo_type=None, # pylint: disable=R0913 self._config.set(name, ExternalsDescription.TAG, tag) - if repo_type: - repo_url = os.path.join('${MANIC_TEST_BARE_REPO_ROOT}', repo_type) + if repo_path: + repo_url = os.path.join('${MANIC_TEST_BARE_REPO_ROOT}', repo_path) self._config.set(name, ExternalsDescription.REPO_URL, repo_url) try: @@ -487,8 +508,8 @@ def update_tag(self, dest_dir, name, tag, repo_type=None, self.write_config(dest_dir, filename) - def update_underspecify_branch_tag(self, dest_dir, name, - filename=CFG_NAME): + def write_without_branch_tag(self, dest_dir, name, + filename=CFG_NAME): """Update a repository protocol, and potentially the remote """ # pylint: disable=R0913 @@ -506,7 +527,7 @@ def update_underspecify_branch_tag(self, dest_dir, name, self.write_config(dest_dir, filename) - def update_underspecify_remove_url(self, dest_dir, name, + def write_without_repo_url(self, dest_dir, name, filename=CFG_NAME): """Update a repository protocol, and potentially the remote """ @@ -519,15 +540,15 @@ def update_underspecify_remove_url(self, dest_dir, name, self.write_config(dest_dir, filename) - def update_protocol(self, dest_dir, name, protocol, repo_type=None, + def write_with_protocol(self, dest_dir, name, protocol, repo_path=None, filename=CFG_NAME): """Update a repository protocol, and potentially the remote """ # pylint: disable=R0913 self._config.set(name, ExternalsDescription.PROTOCOL, protocol) - if repo_type: - repo_url = os.path.join('${MANIC_TEST_BARE_REPO_ROOT}', repo_type) + if repo_path: + repo_url = os.path.join('${MANIC_TEST_BARE_REPO_ROOT}', repo_path) self._config.set(name, ExternalsDescription.REPO_URL, repo_url) self.write_config(dest_dir, filename) @@ -543,6 +564,7 @@ class BaseTestSysCheckout(unittest.TestCase): # cryptic. # pylint: disable=invalid-name + # Command-line args for checkout_externals, used in execute_checkout_in_dir() status_args = ['--status'] checkout_args = [] optional_args = ['--optional'] @@ -575,9 +597,8 @@ def setUp(self): self._checkout = os.path.join(root_dir, 'checkout_externals') # directory where we have test repositories - test_dir = os.path.join(root_dir, 'test') - self._bare_root = os.path.join(test_dir, BARE_REPO_ROOT_NAME) - self._bare_root = os.path.abspath(self._bare_root) + self._bare_root = os.path.abspath( + os.path.join(root_dir, 'test', BARE_REPO_ROOT_NAME)) # set into the environment so var will be expanded in externals files os.environ[MANIC_TEST_BARE_REPO_ROOT] = self._bare_root @@ -596,16 +617,16 @@ def tearDown(self): # return to our common starting point os.chdir(self._return_dir) - def setup_test_repo(self, parent_repo_name, dest_dir_in=None): - """Setup the paths and clone the base test repo + def clone_test_repo(self, parent_repo_name, dest_dir_in=None): + """Clone repo under bare_root into dest_dir_in or local per-test-subdir. + Returns output dir. """ - # unique repo for this test - test_dir_name = self._test_id - print("Test repository name: {0}".format(test_dir_name)) - parent_repo_dir = os.path.join(self._bare_root, parent_repo_name) if dest_dir_in is None: + # unique repo for this test + test_dir_name = self._test_id + print("Test repository name: {0}".format(test_dir_name)) dest_dir = os.path.join(os.environ[MANIC_TEST_TMP_REPO_ROOT], test_dir_name) else: @@ -638,7 +659,7 @@ def _add_file_to_repo(under_test_dir, filename, tracked): os.chdir(cwd) @staticmethod - def execute_cmd_in_dir(under_test_dir, args): + def execute_checkout_in_dir(dirname, args): """Extecute the checkout command in the appropriate repo dir with the specified additional args @@ -649,13 +670,13 @@ def execute_cmd_in_dir(under_test_dir, args): """ cwd = os.getcwd() checkout_path = os.path.abspath('{0}/../../checkout_externals') - os.chdir(under_test_dir) + os.chdir(dirname) cmdline = ['--externals', CFG_NAME, ] cmdline += args repo_root = 'MANIC_TEST_BARE_REPO_ROOT={root}'.format( root=os.environ[MANIC_TEST_BARE_REPO_ROOT]) manual_cmd = ('Test cmd:\npushd {cwd}; {env} {checkout} {args}'.format( - cwd=under_test_dir, env=repo_root, checkout=checkout_path, + cwd=dirname, env=repo_root, checkout=checkout_path, args=' '.join(cmdline))) printlog(manual_cmd) options = checkout.commandline_arguments(cmdline) @@ -667,6 +688,8 @@ def execute_cmd_in_dir(under_test_dir, args): # # Check results for generic perturbation of states # + # 'tree' is a dict with ExternalStatus as values. + # 'name' is a key into that dict. # ---------------------------------------------------------------- def _check_generic_empty_default_required(self, tree, name): self.assertEqual(tree[name].sync_state, ExternalStatus.EMPTY) @@ -702,6 +725,7 @@ def _check_generic_ok_clean_optional(self, tree, name): # # Check results for individual named externals # + # 'tree' is a dict of string to ExternalStatus. # ---------------------------------------------------------------- def _check_simple_tag_empty(self, tree, directory=EXTERNALS_NAME): name = './{0}/simp_tag'.format(directory) @@ -1019,21 +1043,21 @@ def test_container_simple_required(self): """ # create repo - under_test_dir = self.setup_test_repo(CONTAINER_REPO_NAME) - self._generator.container_simple_required(under_test_dir) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO_NAME) + self._generator.container_simple_required(cloned_repo_dir) # status of empty repo - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.status_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.status_args) self._check_container_simple_required_pre_checkout(overall, tree) # checkout - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.checkout_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.checkout_args) self._check_container_simple_required_checkout(overall, tree) # status clean checked out - overall, tree = self.execute_cmd_in_dir(under_test_dir, + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, self.status_args) self._check_container_simple_required_post_checkout(overall, tree) @@ -1049,23 +1073,23 @@ def test_container_nested_required(self): # create repo dest_dir = os.path.join(os.environ[MANIC_TEST_TMP_REPO_ROOT], self._test_id, "test"+str(n)) - under_test_dir = self.setup_test_repo(CONTAINER_REPO_NAME, + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO_NAME, dest_dir_in=dest_dir) - self._generator.container_nested_required(under_test_dir, order) + self._generator.container_nested_required(cloned_repo_dir, order) # status of empty repo - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.status_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.status_args) self._check_container_nested_required_pre_checkout(overall, tree, order) # checkout - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.checkout_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.checkout_args) self._check_container_nested_required_checkout(overall, tree, order) # status clean checked out - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.status_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.status_args) self._check_container_nested_required_post_checkout(overall, tree, order) def test_container_simple_optional(self): @@ -1074,32 +1098,32 @@ def test_container_simple_optional(self): """ # create repo - under_test_dir = self.setup_test_repo(CONTAINER_REPO_NAME) - self._generator.container_simple_optional(under_test_dir) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO_NAME) + self._generator.container_simple_optional(cloned_repo_dir) # check status of empty repo - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.status_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.status_args) self._check_container_simple_optional_pre_checkout(overall, tree) # checkout required - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.checkout_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.checkout_args) self._check_container_simple_optional_checkout(overall, tree) # status - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.status_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.status_args) self._check_container_simple_optional_post_checkout(overall, tree) # checkout optional - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.optional_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.optional_args) self._check_container_simple_optional_post_checkout(overall, tree) # status - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.status_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.status_args) self._check_container_simple_optional_post_optional(overall, tree) def test_container_simple_verbose(self): @@ -1108,17 +1132,17 @@ def test_container_simple_verbose(self): """ # create repo - under_test_dir = self.setup_test_repo(CONTAINER_REPO_NAME) - self._generator.container_simple_required(under_test_dir) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO_NAME) + self._generator.container_simple_required(cloned_repo_dir) # checkout - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.checkout_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.checkout_args) self._check_container_simple_required_checkout(overall, tree) # check verbose status - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.verbose_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.verbose_args) self._check_container_simple_required_post_checkout(overall, tree) def test_container_simple_dirty(self): @@ -1126,28 +1150,28 @@ def test_container_simple_dirty(self): and a dirty status exits gracefully. """ - under_test_dir = self.setup_test_repo(CONTAINER_REPO_NAME) - self._generator.container_simple_required(under_test_dir) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO_NAME) + self._generator.container_simple_required(cloned_repo_dir) # checkout - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.checkout_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.checkout_args) self._check_container_simple_required_checkout(overall, tree) # add a file to the repo tracked = True - self._add_file_to_repo(under_test_dir, 'externals/simp_tag/tmp.txt', + self._add_file_to_repo(cloned_repo_dir, 'externals/simp_tag/tmp.txt', tracked) # checkout: pre-checkout status should be dirty, did not # modify working copy. - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.checkout_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.checkout_args) self._check_container_simple_optional_st_dirty(overall, tree) # verify status is still dirty - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.status_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.status_args) self._check_container_simple_optional_st_dirty(overall, tree) def test_container_simple_untracked(self): @@ -1155,28 +1179,28 @@ def test_container_simple_untracked(self): is not considered 'dirty' and will attempt an update. """ - under_test_dir = self.setup_test_repo(CONTAINER_REPO_NAME) - self._generator.container_simple_required(under_test_dir) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO_NAME) + self._generator.container_simple_required(cloned_repo_dir) # checkout - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.checkout_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.checkout_args) self._check_container_simple_required_checkout(overall, tree) # add a file to the repo tracked = False - self._add_file_to_repo(under_test_dir, 'externals/simp_tag/tmp.txt', + self._add_file_to_repo(cloned_repo_dir, 'externals/simp_tag/tmp.txt', tracked) # checkout: pre-checkout status should be clean, ignoring the # untracked file. - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.checkout_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.checkout_args) self._check_container_simple_required_post_checkout(overall, tree) # verify status is still clean - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.status_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.status_args) self._check_container_simple_required_post_checkout(overall, tree) def test_container_simple_detached_sync(self): @@ -1186,38 +1210,38 @@ def test_container_simple_detached_sync(self): """ # create repo - under_test_dir = self.setup_test_repo(CONTAINER_REPO_NAME) - self._generator.container_simple_required(under_test_dir) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO_NAME) + self._generator.container_simple_required(cloned_repo_dir) # status of empty repo - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.status_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.status_args) self._check_container_simple_required_pre_checkout(overall, tree) # checkout - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.checkout_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.checkout_args) self._check_container_simple_required_checkout(overall, tree) # make a commit on the detached head of the tag and hash externals - self._generator.create_commit(under_test_dir, 'simp_tag') - self._generator.create_commit(under_test_dir, 'simp_hash') - self._generator.create_commit(under_test_dir, 'simp_branch') + self._generator.create_commit(cloned_repo_dir, 'simp_tag') + self._generator.create_commit(cloned_repo_dir, 'simp_hash') + self._generator.create_commit(cloned_repo_dir, 'simp_branch') # status of repo, branch, tag and hash should all be out of sync! - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.status_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.status_args) self._check_container_simple_required_out_of_sync(overall, tree) # checkout - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.checkout_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.checkout_args) # same pre-checkout out of sync status self._check_container_simple_required_out_of_sync(overall, tree) # now status should be in-sync - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.status_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.status_args) self._check_container_simple_required_post_checkout(overall, tree) def test_container_remote_branch(self): @@ -1225,32 +1249,33 @@ def test_container_remote_branch(self): """ # create repo - under_test_dir = self.setup_test_repo(CONTAINER_REPO_NAME) - self._generator.container_simple_required(under_test_dir) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO_NAME) + self._generator.container_simple_required(cloned_repo_dir) # checkout - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.checkout_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.checkout_args) self._check_container_simple_required_checkout(overall, tree) # update the config file to point to a different remote with # the same branch - self._generator.update_branch(under_test_dir, 'simp_branch', - REMOTE_BRANCH_FEATURE2, SIMPLE_FORK_NAME) + self._generator.write_with_git_branch(cloned_repo_dir, name='simp_branch', + branch=REMOTE_BRANCH_FEATURE2, + repo_path=SIMPLE_FORK_NAME) # status of simp_branch should be out of sync - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.status_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.status_args) self._check_container_simple_required_sb_modified(overall, tree) # checkout new externals - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.checkout_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.checkout_args) self._check_container_simple_required_sb_modified(overall, tree) # status should be synced - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.status_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.status_args) self._check_container_simple_required_post_checkout(overall, tree) def test_container_remote_tag_same_branch(self): @@ -1261,33 +1286,33 @@ def test_container_remote_tag_same_branch(self): """ # create repo - under_test_dir = self.setup_test_repo(CONTAINER_REPO_NAME) - self._generator.container_simple_required(under_test_dir) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO_NAME) + self._generator.container_simple_required(cloned_repo_dir) # checkout - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.checkout_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.checkout_args) self._check_container_simple_required_checkout(overall, tree) # update the config file to point to a different remote with # the tag instead of branch. Tag MUST NOT be in the original # repo! - self._generator.update_tag(under_test_dir, 'simp_branch', + self._generator.write_with_tag(cloned_repo_dir, 'simp_branch', 'forked-feature-v1', SIMPLE_FORK_NAME) # status of simp_branch should be out of sync - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.status_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.status_args) self._check_container_simple_required_sb_modified(overall, tree) # checkout new externals - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.checkout_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.checkout_args) self._check_container_simple_required_sb_modified(overall, tree) # status should be synced - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.status_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.status_args) self._check_container_simple_required_post_checkout(overall, tree) def test_container_remote_tag_fetch_all(self): @@ -1299,33 +1324,33 @@ def test_container_remote_tag_fetch_all(self): """ # create repo - under_test_dir = self.setup_test_repo(CONTAINER_REPO_NAME) - self._generator.container_simple_required(under_test_dir) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO_NAME) + self._generator.container_simple_required(cloned_repo_dir) # checkout - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.checkout_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.checkout_args) self._check_container_simple_required_checkout(overall, tree) # update the config file to point to a different remote with # the tag instead of branch. Tag MUST NOT be in the original # repo! - self._generator.update_tag(under_test_dir, 'simp_branch', + self._generator.write_with_tag(cloned_repo_dir, 'simp_branch', 'abandoned-feature', SIMPLE_FORK_NAME) # status of simp_branch should be out of sync - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.status_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.status_args) self._check_container_simple_required_sb_modified(overall, tree) # checkout new externals - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.checkout_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.checkout_args) self._check_container_simple_required_sb_modified(overall, tree) # status should be synced - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.status_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.status_args) self._check_container_simple_required_post_checkout(overall, tree) def test_container_preserve_dot(self): @@ -1334,40 +1359,41 @@ def test_container_preserve_dot(self): """ # create repo - under_test_dir = self.setup_test_repo(CONTAINER_REPO_NAME) - self._generator.container_simple_required(under_test_dir) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO_NAME) + self._generator.container_simple_required(cloned_repo_dir) # checkout - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.checkout_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.checkout_args) self._check_container_simple_required_checkout(overall, tree) # update the config file to point to a different remote with # the same branch - self._generator.update_branch(under_test_dir, 'simp_branch', - REMOTE_BRANCH_FEATURE2, SIMPLE_FORK_NAME) + self._generator.write_with_git_branch(cloned_repo_dir, name='simp_branch', + branch=REMOTE_BRANCH_FEATURE2, + repo_path=SIMPLE_FORK_NAME) # checkout - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.checkout_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.checkout_args) # verify status is clean and unmodified - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.status_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.status_args) self._check_container_simple_required_post_checkout(overall, tree) # update branch to point to a new branch that only exists in # the local fork - self._generator.create_branch(under_test_dir, 'simp_branch', - 'private-feature', with_commit=True) - self._generator.update_branch(under_test_dir, 'simp_branch', - 'private-feature', - SIMPLE_LOCAL_ONLY_NAME) - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.checkout_args) + self._generator.create_branch(cloned_repo_dir, repo_name='simp_branch', + branch='private-feature', with_commit=True) + self._generator.write_with_git_branch(cloned_repo_dir, name='simp_branch', + branch='private-feature', + repo_path=SIMPLE_LOCAL_ONLY_NAME) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.checkout_args) # verify status is clean and unmodified - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.status_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.status_args) self._check_container_simple_required_post_checkout(overall, tree) def test_container_full(self): @@ -1379,98 +1405,99 @@ def test_container_full(self): """ # create the test repository - under_test_dir = self.setup_test_repo(CONTAINER_REPO_NAME) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO_NAME) # create the top level externals file - self._generator.container_full(under_test_dir) + self._generator.container_full(cloned_repo_dir) # inital checkout - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.checkout_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.checkout_args) self._check_container_full_pre_checkout(overall, tree) - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.status_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.status_args) self._check_container_full_post_checkout(overall, tree) # Check existance of some files subrepo_path = os.path.join('externals', 'simp_tag') - self._check_file_exists(under_test_dir, + self._check_file_exists(cloned_repo_dir, os.path.join(subrepo_path, 'readme.txt')) - self._check_file_absent(under_test_dir, os.path.join(subrepo_path, + self._check_file_absent(cloned_repo_dir, os.path.join(subrepo_path, 'simple_subdir', 'subdir_file.txt')) # update the mixed-use repo to point to different branch - self._generator.update_branch(under_test_dir, 'mixed_req', - 'new-feature', MIXED_REPO_NAME) + self._generator.write_with_git_branch(cloned_repo_dir, name='mixed_req', + branch='new-feature', + repo_path=MIXED_REPO_NAME) # check status out of sync for mixed_req, but sub-externals # are still in sync - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.status_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.status_args) self._check_container_full_pre_checkout_ext_change(overall, tree) # run the checkout. Now the mixed use external and it's # sub-exterals should be changed. Returned status is # pre-checkout! - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.checkout_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.checkout_args) self._check_container_full_pre_checkout_ext_change(overall, tree) # check status out of sync for mixed_req, and sub-externals # are in sync. - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.status_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.status_args) self._check_container_full_post_checkout(overall, tree) def test_container_component(self): """Verify that optional component checkout works """ # create the test repository - under_test_dir = self.setup_test_repo(CONTAINER_REPO_NAME) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO_NAME) # create the top level externals file - self._generator.container_full(under_test_dir) + self._generator.container_full(cloned_repo_dir) # inital checkout, first try a nonexistant component argument noref checkout_args = ['simp_opt', 'noref'] checkout_args.extend(self.checkout_args) with self.assertRaises(RuntimeError): - self.execute_cmd_in_dir(under_test_dir, checkout_args) + self.execute_checkout_in_dir(cloned_repo_dir, checkout_args) checkout_args = ['simp_opt'] checkout_args.extend(self.checkout_args) - overall, tree = self.execute_cmd_in_dir(under_test_dir, - checkout_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + checkout_args) - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.status_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.status_args) self._check_container_component_post_checkout(overall, tree) checkout_args.append('simp_branch') - overall, tree = self.execute_cmd_in_dir(under_test_dir, - checkout_args) - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.status_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + checkout_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.status_args) self._check_container_component_post_checkout2(overall, tree) def test_container_exclude_component(self): """Verify that exclude component checkout works """ # create the test repository - under_test_dir = self.setup_test_repo(CONTAINER_REPO_NAME) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO_NAME) # create the top level externals file - self._generator.container_full(under_test_dir) + self._generator.container_full(cloned_repo_dir) # inital checkout, exclude simp_opt checkout_args = ['--exclude', 'simp_opt'] checkout_args.extend(self.checkout_args) - overall, tree = self.execute_cmd_in_dir(under_test_dir, checkout_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, checkout_args) checkout_args.append("--status") - overall, tree = self.execute_cmd_in_dir(under_test_dir, checkout_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, checkout_args) self._check_container_component_post_checkout3(overall, tree) def test_mixed_simple(self): @@ -1480,22 +1507,22 @@ def test_mixed_simple(self): """ #import pdb; pdb.set_trace() # create repository - under_test_dir = self.setup_test_repo(MIXED_REPO_NAME) + cloned_repo_dir = self.clone_test_repo(MIXED_REPO_NAME) # create top level externals file - self._generator.mixed_simple_base(under_test_dir) + self._generator.mixed_simple_base(cloned_repo_dir) # NOTE: sub-externals file is already in the repo so we can # switch branches during testing. Since this is a mixed-repo # serving as the top level container repo, we can't switch # during this test. # checkout - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.checkout_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.checkout_args) self._check_mixed_cont_simple_required_checkout(overall, tree) # verify status is clean and unmodified - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.status_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.status_args) self._check_mixed_cont_simple_required_post_checkout(overall, tree) def test_container_sparse(self): @@ -1504,31 +1531,31 @@ def test_container_sparse(self): """ # create the test repository - under_test_dir = self.setup_test_repo(CONTAINER_REPO_NAME) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO_NAME) # create the top level externals file - self._generator.container_sparse(under_test_dir) + self._generator.container_sparse(cloned_repo_dir) # inital checkout - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.checkout_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.checkout_args) self._check_container_sparse_pre_checkout(overall, tree) - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.status_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.status_args) self._check_container_sparse_post_checkout(overall, tree) # Check existance of some files subrepo_path = os.path.join('externals', 'simp_tag') - self._check_file_exists(under_test_dir, + self._check_file_exists(cloned_repo_dir, os.path.join(subrepo_path, 'readme.txt')) - self._check_file_exists(under_test_dir, os.path.join(subrepo_path, + self._check_file_exists(cloned_repo_dir, os.path.join(subrepo_path, 'simple_subdir', 'subdir_file.txt')) subrepo_path = os.path.join('externals', 'simp_sparse') - self._check_file_exists(under_test_dir, + self._check_file_exists(cloned_repo_dir, os.path.join(subrepo_path, 'readme.txt')) - self._check_file_absent(under_test_dir, os.path.join(subrepo_path, + self._check_file_absent(cloned_repo_dir, os.path.join(subrepo_path, 'simple_subdir', 'subdir_file.txt')) @@ -1625,57 +1652,57 @@ def test_container_simple_svn(self): """ self.skip_if_no_svn_access() # create repo - under_test_dir = self.setup_test_repo(CONTAINER_REPO_NAME) - self._generator.container_simple_svn(under_test_dir) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO_NAME) + self._generator.container_simple_svn(cloned_repo_dir) # checkout - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.checkout_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.checkout_args) # verify status is clean and unmodified - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.status_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.status_args) self._check_container_simple_svn_post_checkout(overall, tree) # update description file to make the tag into a branch and # trigger a switch - self._generator.update_svn_branch(under_test_dir, 'svn_tag', 'trunk') + self._generator.write_with_svn_branch(cloned_repo_dir, 'svn_tag', 'trunk') # checkout - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.checkout_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.checkout_args) # verify status is clean and unmodified - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.status_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.status_args) self._check_container_simple_svn_post_checkout(overall, tree) # add an untracked file to the repo tracked = False - self._add_file_to_repo(under_test_dir, + self._add_file_to_repo(cloned_repo_dir, 'externals/svn_branch/tmp.txt', tracked) # run a no-op checkout: pre-checkout status should be clean, # ignoring the untracked file. - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.checkout_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.checkout_args) self._check_container_simple_svn_post_checkout(overall, tree) # update description file to make the branch into a tag and # trigger a modified sync status - self._generator.update_svn_branch(under_test_dir, 'svn_tag', - 'tags/cesm2.0.beta07') + self._generator.write_with_svn_branch(cloned_repo_dir, 'svn_tag', + 'tags/cesm2.0.beta07') # checkout: pre-checkout status should be clean and modified, # will modify working copy. - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.checkout_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.checkout_args) self._check_container_simple_svn_sb_clean_st_mod(overall, tree) # verify status is still clean and unmodified, last # checkout modified the working dir state. - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.verbose_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.verbose_args) self._check_container_simple_svn_post_checkout(overall, tree) class TestSubrepoCheckout(BaseTestSysCheckout): @@ -1707,7 +1734,7 @@ def setUp(self): self._test_id) self._repo_dir = os.path.join(self._my_test_dir, self._test_repo_name) self._checkout_dir = 'repo_with_submodules' - check_dir = self.setup_test_repo(CONTAINER_REPO_NAME, + check_dir = self.clone_test_repo(CONTAINER_REPO_NAME, dest_dir_in=self._repo_dir) self.assertTrue(self._repo_dir == check_dir) # Add the submodules @@ -1795,7 +1822,7 @@ def create_externals_file(self, name='', filename=CFG_NAME, dest_dir=None, self._checkout_dir, branch=branch_name, path=name, externals=sub_externals, - repo_path=self._repo_dir) + repo_path_abs=self._repo_dir) self._generator.write_config(dest_dir, filename=filename) @@ -1804,11 +1831,11 @@ def idempotence_check(self, checkout_dir): checkout_externals --status does not cause errors""" cwd = os.getcwd() os.chdir(checkout_dir) - overall, _ = self.execute_cmd_in_dir(self._my_test_dir, - self.checkout_args) + overall, _ = self.execute_checkout_in_dir(self._my_test_dir, + self.checkout_args) self.assertTrue(overall == 0) - overall, _ = self.execute_cmd_in_dir(self._my_test_dir, - self.status_args) + overall, _ = self.execute_checkout_in_dir(self._my_test_dir, + self.status_args) self.assertTrue(overall == 0) os.chdir(cwd) @@ -1822,8 +1849,8 @@ def test_submodule_checkout_bare(self): simple_ext_fork_tag = "(tag1)" simple_ext_fork_status = " " self.create_externals_file(branch_name=self._bare_branch_name) - overall, _ = self.execute_cmd_in_dir(self._my_test_dir, - self.checkout_args) + overall, _ = self.execute_checkout_in_dir(self._my_test_dir, + self.checkout_args) self.assertTrue(overall == 0) cwd = os.getcwd() checkout_dir = os.path.join(self._my_test_dir, self._checkout_dir) @@ -1852,8 +1879,8 @@ def test_submodule_checkout_none(self): """ self.create_externals_file(branch_name=self._bare_branch_name, sub_externals="none") - overall, _ = self.execute_cmd_in_dir(self._my_test_dir, - self.checkout_args) + overall, _ = self.execute_checkout_in_dir(self._my_test_dir, + self.checkout_args) self.assertTrue(overall == 0) cwd = os.getcwd() checkout_dir = os.path.join(self._my_test_dir, self._checkout_dir) @@ -1874,8 +1901,8 @@ def test_submodule_checkout_config(self): # pylint: disable=too-many-locals status_check = "-" # Not checked out as submodule self.create_externals_file(branch_name=self._config_branch_name, sub_externals=self._container_extern_name) - overall, _ = self.execute_cmd_in_dir(self._my_test_dir, - self.checkout_args) + overall, _ = self.execute_checkout_in_dir(self._my_test_dir, + self.checkout_args) self.assertTrue(overall == 0) cwd = os.getcwd() checkout_dir = os.path.join(self._my_test_dir, self._checkout_dir) @@ -1938,17 +1965,17 @@ def test_error_unknown_protocol(self): """ # create repo - under_test_dir = self.setup_test_repo(CONTAINER_REPO_NAME) - self._generator.container_simple_required(under_test_dir) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO_NAME) + self._generator.container_simple_required(cloned_repo_dir) # update the config file to point to a different remote with # the tag instead of branch. Tag MUST NOT be in the original # repo! - self._generator.update_protocol(under_test_dir, 'simp_branch', + self._generator.write_with_protocol(cloned_repo_dir, 'simp_branch', 'this-protocol-does-not-exist') with self.assertRaises(RuntimeError): - self.execute_cmd_in_dir(under_test_dir, self.checkout_args) + self.execute_checkout_in_dir(cloned_repo_dir, self.checkout_args) def test_error_switch_protocol(self): """Verify that a runtime error is raised when the user switches @@ -1959,15 +1986,15 @@ def test_error_switch_protocol(self): """ # create repo - under_test_dir = self.setup_test_repo(CONTAINER_REPO_NAME) - self._generator.container_simple_required(under_test_dir) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO_NAME) + self._generator.container_simple_required(cloned_repo_dir) # update the config file to point to a different remote with # the tag instead of branch. Tag MUST NOT be in the original # repo! - self._generator.update_protocol(under_test_dir, 'simp_branch', 'svn') + self._generator.write_with_protocol(cloned_repo_dir, 'simp_branch', 'svn') with self.assertRaises(RuntimeError): - self.execute_cmd_in_dir(under_test_dir, self.checkout_args) + self.execute_checkout_in_dir(cloned_repo_dir, self.checkout_args) def test_error_unknown_tag(self): """Verify that a runtime error is raised when the user specified tag @@ -1975,17 +2002,17 @@ def test_error_unknown_tag(self): """ # create repo - under_test_dir = self.setup_test_repo(CONTAINER_REPO_NAME) - self._generator.container_simple_required(under_test_dir) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO_NAME) + self._generator.container_simple_required(cloned_repo_dir) # update the config file to point to a different remote with # the tag instead of branch. Tag MUST NOT be in the original # repo! - self._generator.update_tag(under_test_dir, 'simp_branch', + self._generator.write_with_tag(cloned_repo_dir, 'simp_branch', 'this-tag-does-not-exist', SIMPLE_REPO_NAME) with self.assertRaises(RuntimeError): - self.execute_cmd_in_dir(under_test_dir, self.checkout_args) + self.execute_checkout_in_dir(cloned_repo_dir, self.checkout_args) def test_error_overspecify_tag_branch(self): """Verify that a runtime error is raised when the user specified both @@ -1993,18 +2020,18 @@ def test_error_overspecify_tag_branch(self): """ # create repo - under_test_dir = self.setup_test_repo(CONTAINER_REPO_NAME) - self._generator.container_simple_required(under_test_dir) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO_NAME) + self._generator.container_simple_required(cloned_repo_dir) # update the config file to point to a different remote with # the tag instead of branch. Tag MUST NOT be in the original # repo! - self._generator.update_tag(under_test_dir, 'simp_branch', - 'this-tag-does-not-exist', SIMPLE_REPO_NAME, - remove_branch=False) + self._generator.write_with_tag(cloned_repo_dir, 'simp_branch', + 'this-tag-does-not-exist', SIMPLE_REPO_NAME, + remove_branch=False) with self.assertRaises(RuntimeError): - self.execute_cmd_in_dir(under_test_dir, self.checkout_args) + self.execute_checkout_in_dir(cloned_repo_dir, self.checkout_args) def test_error_underspecify_tag_branch(self): """Verify that a runtime error is raised when the user specified @@ -2012,17 +2039,17 @@ def test_error_underspecify_tag_branch(self): """ # create repo - under_test_dir = self.setup_test_repo(CONTAINER_REPO_NAME) - self._generator.container_simple_required(under_test_dir) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO_NAME) + self._generator.container_simple_required(cloned_repo_dir) # update the config file to point to a different remote with # the tag instead of branch. Tag MUST NOT be in the original # repo! - self._generator.update_underspecify_branch_tag(under_test_dir, - 'simp_branch') + self._generator.write_without_branch_tag(cloned_repo_dir, + 'simp_branch') with self.assertRaises(RuntimeError): - self.execute_cmd_in_dir(under_test_dir, self.checkout_args) + self.execute_checkout_in_dir(cloned_repo_dir, self.checkout_args) def test_error_missing_url(self): """Verify that a runtime error is raised when the user specified @@ -2030,17 +2057,17 @@ def test_error_missing_url(self): """ # create repo - under_test_dir = self.setup_test_repo(CONTAINER_REPO_NAME) - self._generator.container_simple_required(under_test_dir) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO_NAME) + self._generator.container_simple_required(cloned_repo_dir) # update the config file to point to a different remote with # the tag instead of branch. Tag MUST NOT be in the original # repo! - self._generator.update_underspecify_remove_url(under_test_dir, + self._generator.write_without_repo_url(cloned_repo_dir, 'simp_branch') with self.assertRaises(RuntimeError): - self.execute_cmd_in_dir(under_test_dir, self.checkout_args) + self.execute_checkout_in_dir(cloned_repo_dir, self.checkout_args) if __name__ == '__main__': diff --git a/test/test_unit_repository_svn.py b/test/test_unit_repository_svn.py old mode 100644 new mode 100755 index 41b173bf3d..d9309df7f6 --- a/test/test_unit_repository_svn.py +++ b/test/test_unit_repository_svn.py @@ -60,7 +60,7 @@ def setUp(self): self._name = 'component' rdata = {ExternalsDescription.PROTOCOL: 'svn', ExternalsDescription.REPO_URL: - 'https://svn-ccsm-models.cgd.ucar.edu/', + 'https://svn-ccsm-models.cgd.ucar.edu', ExternalsDescription.TAG: 'mosart/trunk_tags/mosart1_0_26', }