input_osv3: use git_commit_depth and container.yaml from user_params

Now that osbs-client passes the git commit depth and container.yaml
from the git repo, read those values from user_params and pass them
back through provider_params.

LazyGit will attempt to do a single branch, shallow clone if a branch
and commit depth is provided in provider_params.

Signed-off-by: Mark Langsdorf <[email protected]>

atomic_reactor/plugins/input_osv3.py

@@ -44,15 +44,17 @@ def __init__(self, **kwargs):
         # call parent constructor
         super(OSv3InputPlugin, self).__init__(**kwargs)
 
-    def get_plugins_with_user_params(self, build_json, user_params):
+    def validate_user_data(self, user_params):
+        # make sure the input json is valid
+        read_yaml(user_params, 'schemas/user_params.json')
+        return json.loads(user_params)
+
+    def get_plugins_with_user_data(self, user_params, user_data):
         #  get the reactor config map and derive an osbs instance from it
 
         from osbs.api import OSBS
         from osbs.conf import Configuration
 
-        # make sure the input json is valid
-        read_yaml(user_params, 'schemas/user_params.json')
-        user_data = json.loads(user_params)
         reactor_config_override = user_data.get('reactor_config_override')
         if reactor_config_override:
             read_yaml(json.dumps(reactor_config_override), 'schemas/config.json')
@@ -182,9 +184,16 @@ def run(self):
         self.target_registry = os.environ.get('OUTPUT_REGISTRY', None)
         self.reactor_env = None
 
+        git_commit_depth = None
+        container_yaml = {}
+        git_branch = None
         try:
             user_params = os.environ['USER_PARAMS']
-            self.plugins_json = self.get_plugins_with_user_params(build_json, user_params)
+            user_data = self.validate_user_data(user_params)
+            git_commit_depth = user_data.get('git_commit_depth', None)
+            container_yaml = user_data.get('container_yaml', None)
+            git_branch = user_data.get('git_branch', None)
+            self.plugins_json = self.get_plugins_with_user_data(user_params, user_data)
             # if we get the USER_PARAMS, we'd better get the REACTOR_CONFIG too
             reactor_config_map = os.environ['REACTOR_CONFIG']
             self.reactor_env = read_yaml(reactor_config_map, 'schemas/config.json')
@@ -202,7 +211,12 @@ def run(self):
             'source': {
                 'provider': 'git',
                 'uri': git_url,
-                'provider_params': {'git_commit': git_ref}
+                'provider_params': {
+                    'git_commit': git_ref,
+                    'container_yaml': container_yaml,
+                    'git_commit_depth': git_commit_depth,
+                    'git_branch': git_branch,
+                },
             },
             'image': image,
             'openshift_build_selflink': build_json.get('metadata', {}).get('selfLink', None)

atomic_reactor/schemas/user_params.json

@@ -5,6 +5,10 @@
 
   "type": ["object", "null"],
   "properties": {
+    "additional_tags": {
+      "type": "array",
+      "items": {"type": "string"}
+    },
     "arrangement_version": {"type": "integer"},
     "base_image": {"type": "string"},
     "build_from": {"type": "string"},
@@ -17,10 +21,12 @@
       "type": "array",
       "items": {"type": "integer"}
     },
+    "container_yaml": {"$ref": "container.json"},
     "customize_conf": {"type": "string"},
     "filesystem_koji_task_id": {"type": "integer"},
     "flatpak": {"type": "boolean"},
     "git_branch": {"type": "string"},
+    "git_commit_depth": {"type": "integer"},
     "git_ref": {"type": "string"},
     "git_uri": {"type": "string"},
     "image_tag": {"type": "string"},
@@ -61,6 +67,7 @@
     "release": {"type": "string"},
     "scratch": {"type": "boolean"},
     "signing_intent": {"type": "string"},
+    "tags_from_yaml": {"type": "boolean"},
     "trigger_imagestreamtag": {"type": "string"},
     "user": {"type": "string"},
     "yum_repourls": {

atomic_reactor/source.py

@@ -35,21 +35,24 @@
 class SourceConfig(object):
     """ read container.yaml file from build source and store as attrs """
 
-    def __init__(self, build_path):
-        self.data = {}
+    def __init__(self, build_path, provider_params=None):
+        git_params = provider_params or {}
+        # validated in input_osv3 if its here at all
+        self.data = git_params.get('container_yaml', {})
         self.file_path = os.path.join(build_path, REPO_CONTAINER_CONFIG)
-        if os.path.exists(self.file_path):
-            try:
-                # read file and validate against schema
-                self.data = read_yaml_from_file_path(
-                    self.file_path, 'schemas/container.json'
-                ) or {}
-            except Exception:
-                logger.exception(
-                    "Failed to load and validate source config YAML from %s",
-                    self.file_path
-                )
-                raise
+        if not self.data:
+            if os.path.exists(self.file_path):
+                try:
+                    # read file and validate against schema
+                    self.data = read_yaml_from_file_path(
+                        self.file_path, 'schemas/container.json'
+                    ) or {}
+                except Exception:
+                    logger.exception(
+                        "Failed to load and validate source config YAML from %s",
+                        self.file_path
+                    )
+                    raise
 
         self.autorebuild = self.data.get('autorebuild') or {}
         self.flatpak = self.data.get('flatpak')
@@ -98,7 +101,7 @@ def get_build_file_path(self):
     @property
     def config(self):
         # contents of container.yaml
-        self._config = self._config or SourceConfig(self.path)
+        self._config = self._config or SourceConfig(self.path, self.provider_params)
         return self._config
 
     def remove_tmpdir(self):
@@ -114,7 +117,10 @@ def __init__(self, provider, uri, dockerfile_path=None, provider_params=None, tm
         super(GitSource, self).__init__(provider, uri, dockerfile_path,
                                         provider_params, tmpdir)
         self.git_commit = self.provider_params.get('git_commit', None)
-        self.lg = util.LazyGit(self.uri, self.git_commit, self.source_path)
+        branch = self.provider_params.get('git_commit', None)
+        depth = self.provider_params.get('git_commit_depth', None)
+        self.lg = util.LazyGit(self.uri, self.git_commit, self.source_path, branch=branch,
+                               depth=depth)
 
     @property
     def commit_id(self):

atomic_reactor/util.py

@@ -303,14 +303,17 @@ def wait_for_command(logs_generator):
     return cr
 
 
-def clone_git_repo(git_url, target_dir, commit=None, retry_times=GIT_MAX_RETRIES):
+def clone_git_repo(git_url, target_dir, commit=None, retry_times=GIT_MAX_RETRIES, branch=None,
+                   depth=None):
     """
     clone provided git repo to target_dir, optionally checkout provided commit
 
     :param git_url: str, git repo to clone
     :param target_dir: str, filesystem path where the repo should be cloned
     :param commit: str, commit to checkout, SHA-1 or ref
     :param retry_times: int, number of retries for git clone
+    :param branch: str, optional branch of the commit, required if depth is provided
+    :param depth: int, optional expected depth
     :return: str, commit ID of HEAD
     """
     retry_delay = GIT_BACKOFF_FACTOR
@@ -320,14 +323,24 @@ def clone_git_repo(git_url, target_dir, commit=None, retry_times=GIT_MAX_RETRIES
     logger.debug("url = '%s', dir = '%s', commit = '%s'",
                  git_url, target_dir, commit)
 
-    cmd = ["git", "clone", git_url, quote(target_dir)]
+    cmd = ["git", "clone"]
+    if branch:
+        cmd += ["-b", branch, "--single-branch"]
+        if depth:
+            cmd += ["--depth", '{0}'.format(depth)]
+    elif depth:
+        logger.warning("branch not provided for %s, depth setting ignored", git_url)
+
+    cmd += [git_url, quote(target_dir)]
 
     logger.debug("cloning '%s'", cmd)
+    repo_commit = ''
     for counter in range(retry_times + 1):
         try:
             # we are using check_output, even though we aren't using
             # the return value, but we will get 'output' in exception
             subprocess.check_output(cmd, stderr=subprocess.STDOUT)
+            repo_commit = reset_git_repo(target_dir, commit, depth and branch)
             break
         except subprocess.CalledProcessError as exc:
             if counter != retry_times:
@@ -336,20 +349,35 @@ def clone_git_repo(git_url, target_dir, commit=None, retry_times=GIT_MAX_RETRIES
             else:
                 raise
 
-    return reset_git_repo(target_dir, commit)
+    return repo_commit
 
 
-def reset_git_repo(target_dir, git_reference):
+def reset_git_repo(target_dir, git_reference, retry_deeper=False):
     """
     hard reset git clone in target_dir to given git_reference
 
     :param target_dir: str, filesystem path where the repo is cloned
     :param git_reference: str, any valid git reference
+    :param retry_deeper: bool, if the repo was cloned with shallow, attempt fetch --deepen if
+                         git_reference not found
     :return: str, commit ID of HEAD
     """
-    cmd = ["git", "reset", "--hard", git_reference]
-    logger.debug("Resetting current HEAD: '%s'", cmd)
-    subprocess.check_call(cmd, cwd=target_dir)
+    deepen = 1
+    for _ in range(1, 10):
+        try:
+            cmd = ["git", "reset", "--hard", git_reference]
+            logger.debug("Resetting current HEAD: '%s'", cmd)
+            subprocess.check_call(cmd, cwd=target_dir)
+            break
+        except subprocess.CalledProcessError:
+            if not retry_deeper:
+                raise KeyError('cannot find commit %s in repo %s', git_reference, target_dir)
+            cmd = ["git", "fetch", "--deepen", "{0}".format(deepen)]
+            subprocess.check_call(cmd, cwd=target_dir)
+            deepen *= 2
+    else:
+        raise KeyError('cannot find commit %s in repo %s', git_reference, target_dir)
+
     cmd = ["git", "rev-parse", "HEAD"]
     logger.debug("getting SHA-1 of provided ref '%s'", cmd)
     commit_id = subprocess.check_output(cmd, cwd=target_dir, universal_newlines=True)
@@ -371,14 +399,16 @@ class LazyGit(object):
         lazy_git = LazyGit(git_url="...", tmpdir=tmp_dir)
         lazy_git.git_path
     """
-    def __init__(self, git_url, commit=None, tmpdir=None):
+    def __init__(self, git_url, commit=None, tmpdir=None, branch=None, depth=None):
         self.git_url = git_url
         # provided commit ID/reference to check out
         self.commit = commit
         # commit ID of HEAD; we'll figure this out ourselves
         self._commit_id = None
         self.provided_tmpdir = tmpdir
         self._git_path = None
+        self._branch = None
+        self._depth = depth
 
     @property
     def _tmpdir(self):
@@ -391,7 +421,8 @@ def commit_id(self):
     @property
     def git_path(self):
         if self._git_path is None:
-            self._commit_id = clone_git_repo(self.git_url, self._tmpdir, self.commit)
+            self._commit_id = clone_git_repo(self.git_url, self._tmpdir, self.commit,
+                                             branch=self._branch, depth=self._depth)
             self._git_path = self._tmpdir
         return self._git_path
 

tests/constants.py

@@ -21,6 +21,8 @@
 DOCKERFILE_FILENAME = 'Dockerfile'
 DOCKERFILE_GIT = "https://github.com/TomasTomecek/docker-hello-world.git"
 DOCKERFILE_SHA1 = "6e592f1420efcd331cd28b360a7e02f669caf540"
+DOCKERFILE_INIT_SHA1 = "04523782eeb1e6c960b12f2f6fc887aa7cf76290"
+DOCKERFILE_BRANCH = "error-build"
 DOCKERFILE_OK_PATH = os.path.join(FILES, 'docker-hello-world')
 DOCKERFILE_MULTISTAGE_PATH = os.path.join(FILES, 'docker-hello-world-multistage')
 DOCKERFILE_MULTISTAGE_SCRATCH_PATH = os.path.join(FILES, 'docker-hello-world-multistage-scratch')

tests/test_util.py

@@ -57,7 +57,8 @@
                                  dump_stacktraces, setup_introspection_signal_handler,
                                  DigestCollector)
 from atomic_reactor import util
-from tests.constants import (DOCKERFILE_GIT, DOCKERFILE_SHA1,
+from tests.constants import (DOCKERFILE_GIT, DOCKERFILE_SHA1, DOCKERFILE_INIT_SHA1,
+                             DOCKERFILE_BRANCH,
                              INPUT_IMAGE, MOCK, MOCK_SOURCE,
                              REACTOR_CONFIG_MAP)
 import atomic_reactor.util
@@ -179,15 +180,40 @@ def test_wait_for_command():
     assert wait_for_command(logs_gen) is not None
 
 
+@pytest.mark.parametrize(('commit', 'branch', 'depth'), [
+    (None, None, None),
+    (DOCKERFILE_SHA1, None, None),
+    (DOCKERFILE_SHA1, DOCKERFILE_BRANCH, 1),
+    (DOCKERFILE_INIT_SHA1, DOCKERFILE_BRANCH, 1),
+    (DOCKERFILE_SHA1, None, 1),
+])
 @requires_internet
-def test_clone_git_repo(tmpdir):
+def test_clone_git_repo(tmpdir, commit, branch, depth):
     tmpdir_path = str(tmpdir.realpath())
-    commit_id = clone_git_repo(DOCKERFILE_GIT, tmpdir_path)
+    commit_id = clone_git_repo(DOCKERFILE_GIT, tmpdir_path, commit=commit, branch=branch,
+                               depth=depth)
     assert commit_id is not None
+    if commit:
+        assert commit == commit_id
     assert len(commit_id) == 40  # current git hashes are this long
     assert os.path.isdir(os.path.join(tmpdir_path, '.git'))
 
 
+@pytest.mark.parametrize(('commit', 'branch', 'depth'), [
+    ("bad", None, None),
+    ("bad", DOCKERFILE_BRANCH, 1),
+    ("bad", DOCKERFILE_BRANCH, 1),
+    ("bad", None, 1),
+])
+@requires_internet
+def test_clone_git_repo_failure(tmpdir, commit, branch, depth):
+    tmpdir_path = str(tmpdir.realpath())
+    with pytest.raises(KeyError) as exc:
+        clone_git_repo(DOCKERFILE_GIT, tmpdir_path, retry_times=1,
+                       commit=commit, branch=branch, depth=depth)
+        assert 'cannot find commit' in str(exc)
+
+
 class TestCommandResult(object):
     @pytest.mark.parametrize(('item', 'expected'), [
         ({"stream": "Step 0 : FROM ebbc51b7dfa5bcd993a[...]"},
@@ -202,16 +228,6 @@ def test_parse_item(self, item, expected):
         assert cr.logs == [expected]
 
 
-@requires_internet
-def test_clone_git_repo_by_sha1(tmpdir):
-    tmpdir_path = str(tmpdir.realpath())
-    commit_id = clone_git_repo(DOCKERFILE_GIT, tmpdir_path, commit=DOCKERFILE_SHA1)
-    assert commit_id is not None
-    assert commit_id == DOCKERFILE_SHA1
-    assert len(commit_id) == 40  # current git hashes are this long
-    assert os.path.isdir(os.path.join(tmpdir_path, '.git'))
-
-
 BUILD_FILE_CONTENTS_DOCKER = {
     "Dockerfile": "",
     "container.yaml": "",