Squashed 'manage_externals/' changes from 7b6d92ef6..0f884bfec

0f884bfec Merge pull request ESCOMP#205 from jedwards4b/sunset_svn_git_access
82a5edf79 merge in billsacks:svn_testing_no_github
17532c160 Use a local svn repo for testing
9c904341a different method to determine if in tests
539952ebd remove debug print statement
cc5434fa7 fix submodule testing
1d7f28840 remove broken tests
04e94a519 provide a meaningful error message
38bcc0a8c Merge pull request ESCOMP#201 from jedwards4b/partial_match
b4466a5aa remove debug print statement
c3cf3ec35 fix issue with partial branch match

git-subtree-dir: manage_externals
git-subtree-split: 0f884bfec8e43d0c02261de858d6ec3f6d855e51

manic/repository_git.py

@@ -7,6 +7,7 @@
 
 import copy
 import os
+import sys
 
 from .global_constants import EMPTY_STR, LOCAL_PATH_INDICATOR
 from .global_constants import VERBOSITY_VERBOSE
@@ -380,7 +381,6 @@ def _check_for_valid_ref(self, ref, remote_name, dirname):
         is_tag = self._ref_is_tag(ref, dirname)
         is_branch = self._ref_is_branch(ref, remote_name, dirname)
         is_hash = self._ref_is_hash(ref, dirname)
-
         is_valid = is_tag or is_branch or is_hash
         if not is_valid:
             msg = ('In repo "{0}": reference "{1}" does not appear to be a '
@@ -710,7 +710,10 @@ def _git_lsremote_branch(ref, remote_name, dirname):
         cmd = ('git -C {dirname} ls-remote --exit-code --heads '
                '{remote_name} {ref}').format(
                    dirname=dirname, remote_name=remote_name, ref=ref).split()
-        status = execute_subprocess(cmd, status_to_caller=True)
+        status, output = execute_subprocess(cmd, status_to_caller=True, output_to_caller=True)
+        if not status and not f"refs/heads/{ref}" in output:
+            # In this case the ref is contained in the branch name but is not the complete branch name
+            return -1
         return status
 
     @staticmethod
@@ -837,12 +840,19 @@ def _git_update_submodules(verbosity, dirname):
         """Run git submodule update for the side effect of updating this
         repo's submodules.
         """
+        # due to https://vielmetti.typepad.com/logbook/2022/10/git-security-fixes-lead-to-fatal-transport-file-not-allowed-error-in-ci-systems-cve-2022-39253.html
+        # submodules from file doesn't work without overriding the protocol, this is done
+        # for testing submodule support but should not be done in practice
+        file_protocol = ""
+        if 'unittest' in sys.modules.keys():
+            file_protocol = "-c protocol.file.allow=always"
+
         # First, verify that we have a .gitmodules file
         if os.path.exists(
                 os.path.join(dirname,
                              ExternalsDescription.GIT_SUBMODULES_FILENAME)):
-            cmd = ('git -C {dirname} submodule update --init --recursive'
-                   .format(dirname=dirname)).split()
+            cmd = ('git {file_protocol} -C {dirname} submodule update --init --recursive'
+                   .format(file_protocol=file_protocol, dirname=dirname)).split()
             if verbosity >= VERBOSITY_VERBOSE:
                 printlog('    {0}'.format(' '.join(cmd)))
 

manic/repository_svn.py

@@ -42,6 +42,9 @@ def __init__(self, component_name, repo, ignore_ancestry=False):
         Parse repo (a <repo> XML element).
         """
         Repository.__init__(self, component_name, repo)
+        if 'github.com' in self._url:
+            msg = "SVN access to github.com is no longer supported"
+            fatal_error(msg)
         self._ignore_ancestry = ignore_ancestry
         if self._url.endswith('/'):
             # there is already a '/' separator in the URL; no need to add another

test/repos/README.md

@@ -1,6 +1,6 @@
-Git repositories for testing git-related behavior.  For usage and terminology notes, see test/test_sys_checkout.py.
+Git and svn repositories for testing git and svn-related behavior.  For usage and terminology notes, see test/test_sys_checkout.py.
 
-To list files and view file contents at HEAD:
+For git repos: To list files and view file contents at HEAD:
 ```
 cd <repo_dir>
 git ls-tree --full-tree -r --name-only HEAD

test/repos/simple-ext.svn/README.txt

@@ -0,0 +1,5 @@
+This is a Subversion repository; use the 'svnadmin' and 'svnlook' 
+tools to examine it.  Do not add, delete, or modify files here 
+unless you know how to avoid corrupting the repository.
+
+Visit http://subversion.apache.org/ for more information.

test/repos/simple-ext.svn/conf/authz

@@ -0,0 +1,32 @@
+### This file is an example authorization file for svnserve.
+### Its format is identical to that of mod_authz_svn authorization
+### files.
+### As shown below each section defines authorizations for the path and
+### (optional) repository specified by the section name.
+### The authorizations follow. An authorization line can refer to:
+###  - a single user,
+###  - a group of users defined in a special [groups] section,
+###  - an alias defined in a special [aliases] section,
+###  - all authenticated users, using the '$authenticated' token,
+###  - only anonymous users, using the '$anonymous' token,
+###  - anyone, using the '*' wildcard.
+###
+### A match can be inverted by prefixing the rule with '~'. Rules can
+### grant read ('r') access, read-write ('rw') access, or no access
+### ('').
+
+[aliases]
+# joe = /C=XZ/ST=Dessert/L=Snake City/O=Snake Oil, Ltd./OU=Research Institute/CN=Joe Average
+
+[groups]
+# harry_and_sally = harry,sally
+# harry_sally_and_joe = harry,sally,&joe
+
+# [/foo/bar]
+# harry = rw
+# &joe = r
+# * =
+
+# [repository:/baz/fuz]
+# @harry_and_sally = rw
+# * = r

test/repos/simple-ext.svn/conf/hooks-env.tmpl

@@ -0,0 +1,19 @@
+### This file is an example hook script environment configuration file.
+### Hook scripts run in an empty environment by default.
+### As shown below each section defines environment variables for a
+### particular hook script. The [default] section defines environment
+### variables for all hook scripts, unless overridden by a hook-specific
+### section.
+
+### This example configures a UTF-8 locale for all hook scripts, so that 
+### special characters, such as umlauts, may be printed to stderr.
+### If UTF-8 is used with a mod_dav_svn server, the SVNUseUTF8 option must
+### also be set to 'yes' in httpd.conf.
+### With svnserve, the LANG environment variable of the svnserve process
+### must be set to the same value as given here.
+[default]
+LANG = en_US.UTF-8
+
+### This sets the PATH environment variable for the pre-commit hook.
+[pre-commit]
+PATH = /usr/local/bin:/usr/bin:/usr/sbin

test/repos/simple-ext.svn/conf/passwd

@@ -0,0 +1,8 @@
+### This file is an example password file for svnserve.
+### Its format is similar to that of svnserve.conf. As shown in the
+### example below it contains one section labelled [users].
+### The name and password for each user follow, one account per line.
+
+[users]
+# harry = harryssecret
+# sally = sallyssecret

test/repos/simple-ext.svn/conf/svnserve.conf

@@ -0,0 +1,81 @@
+### This file controls the configuration of the svnserve daemon, if you
+### use it to allow access to this repository.  (If you only allow
+### access through http: and/or file: URLs, then this file is
+### irrelevant.)
+
+### Visit http://subversion.apache.org/ for more information.
+
+[general]
+### The anon-access and auth-access options control access to the
+### repository for unauthenticated (a.k.a. anonymous) users and
+### authenticated users, respectively.
+### Valid values are "write", "read", and "none".
+### Setting the value to "none" prohibits both reading and writing;
+### "read" allows read-only access, and "write" allows complete 
+### read/write access to the repository.
+### The sample settings below are the defaults and specify that anonymous
+### users have read-only access to the repository, while authenticated
+### users have read and write access to the repository.
+# anon-access = read
+# auth-access = write
+### The password-db option controls the location of the password
+### database file.  Unless you specify a path starting with a /,
+### the file's location is relative to the directory containing
+### this configuration file.
+### If SASL is enabled (see below), this file will NOT be used.
+### Uncomment the line below to use the default password file.
+# password-db = passwd
+### The authz-db option controls the location of the authorization
+### rules for path-based access control.  Unless you specify a path
+### starting with a /, the file's location is relative to the
+### directory containing this file.  The specified path may be a
+### repository relative URL (^/) or an absolute file:// URL to a text
+### file in a Subversion repository.  If you don't specify an authz-db,
+### no path-based access control is done.
+### Uncomment the line below to use the default authorization file.
+# authz-db = authz
+### The groups-db option controls the location of the file with the
+### group definitions and allows maintaining groups separately from the
+### authorization rules.  The groups-db file is of the same format as the
+### authz-db file and should contain a single [groups] section with the
+### group definitions.  If the option is enabled, the authz-db file cannot
+### contain a [groups] section.  Unless you specify a path starting with
+### a /, the file's location is relative to the directory containing this
+### file.  The specified path may be a repository relative URL (^/) or an
+### absolute file:// URL to a text file in a Subversion repository.
+### This option is not being used by default.
+# groups-db = groups
+### This option specifies the authentication realm of the repository.
+### If two repositories have the same authentication realm, they should
+### have the same password database, and vice versa.  The default realm
+### is repository's uuid.
+# realm = My First Repository
+### The force-username-case option causes svnserve to case-normalize
+### usernames before comparing them against the authorization rules in the
+### authz-db file configured above.  Valid values are "upper" (to upper-
+### case the usernames), "lower" (to lowercase the usernames), and
+### "none" (to compare usernames as-is without case conversion, which
+### is the default behavior).
+# force-username-case = none
+### The hooks-env options specifies a path to the hook script environment 
+### configuration file. This option overrides the per-repository default
+### and can be used to configure the hook script environment for multiple 
+### repositories in a single file, if an absolute path is specified.
+### Unless you specify an absolute path, the file's location is relative
+### to the directory containing this file.
+# hooks-env = hooks-env
+
+[sasl]
+### This option specifies whether you want to use the Cyrus SASL
+### library for authentication. Default is false.
+### Enabling this option requires svnserve to have been built with Cyrus
+### SASL support; to check, run 'svnserve --version' and look for a line
+### reading 'Cyrus SASL authentication is available.'
+# use-sasl = true
+### These options specify the desired strength of the security layer
+### that you want SASL to provide. 0 means no encryption, 1 means
+### integrity-checking only, values larger than 1 are correlated
+### to the effective key length for encryption (e.g. 128 means 128-bit
+### encryption). The values below are the defaults.
+# min-encryption = 0
+# max-encryption = 256

test/repos/simple-ext.svn/db/current

@@ -0,0 +1 @@
+3

test/repos/simple-ext.svn/db/format

@@ -0,0 +1,3 @@
+8
+layout sharded 1000
+addressing logical

test/repos/simple-ext.svn/db/fs-type

@@ -0,0 +1 @@
+fsfs