test: add coverage for git repo api

* indicate that ordered releases method on codebase now returns a list and
  add public_releases() which returns a queryset

django/core/settings/test.py

@@ -17,7 +17,7 @@
 SHARE_DIR = path.realpath("library/tests/tmp")
 LIBRARY_ROOT = path.join(SHARE_DIR, "library")
 LIBRARY_PREVIOUS_ROOT = path.join(SHARE_DIR, ".latest")
-REPOSITORY_ROOT = path.join(BASE_DIR, "repository")
+REPOSITORY_ROOT = path.join(SHARE_DIR, "repository")
 BACKUP_ROOT = path.join(SHARE_DIR, "backups")
 BORG_ROOT = path.join(BACKUP_ROOT, "repo")
 EXTRACT_ROOT = path.join(SHARE_DIR, "extract")

django/core/tests/base.py

@@ -171,5 +171,13 @@ def initialize_test_shared_folders():
     )
 
 
+def clear_test_shared_folder(dir=settings.REPOSITORY_ROOT):
+    for fs in os.scandir(dir):
+        if fs.is_dir():
+            shutil.rmtree(os.path.join(dir, fs.name), ignore_errors=True)
+        elif fs.is_file():
+            os.remove(os.path.join(dir, fs.name))
+
+
 def destroy_test_shared_folders():
     shutil.rmtree(settings.SHARE_DIR, ignore_errors=True)

django/curator/tests/test_dump_restore.py

@@ -19,7 +19,7 @@
 from core.tests.base import EventFactory, JobFactory
 from library.fs import import_archive
 from library.models import Codebase
-from library.tests.base import CodebaseFactory
+from library.tests.base import CodebaseFactory, TEST_SAMPLES_DIR
 
 logger = logging.getLogger(__name__)
 
@@ -51,7 +51,7 @@ def setUp(self):
         fs_api = self.release.get_fs_api()
         import_archive(
             codebase_release=self.release,
-            nested_code_folder_name="library/tests/archives/nestedcode",
+            nested_code_folder_name=TEST_SAMPLES_DIR / "archives" / "nestedcode",
             fs_api=fs_api,
         )
 

django/library/fs.py

@@ -906,7 +906,9 @@ def build(self):
         mirror for the first time or need to rebuild the entire history
         """
         try:
-            releases = self.codebase.ordered_releases()
+            releases = self.codebase.ordered_releases_list()
+            if not releases:
+                raise ValidationError("Must have at least one public release to build from")
             with self.use_temporary_repo():
                 self.initialize()
                 for release in releases:

django/library/jinja2/library/codebases/releases/retrieve.jinja

@@ -502,7 +502,7 @@
                 </tr>
                 </thead>
                 <tbody>
-                    {% for related_release in codebase.ordered_releases(has_change_perm, asc=False) %}
+                    {% for related_release in codebase.ordered_releases_list(has_change_perm, asc=False) %}
                         <tr>
                             <td><a href='{{ related_release.get_absolute_url() }}'>{{ related_release.version_number }}</a></td>
                             <td>

django/library/migrations/0027_codebasegitmirror_codebase_git_mirror.py → django/library/migrations/0029_codebasegitmirror_codebase_git_mirror.py

@@ -1,4 +1,4 @@
-# Generated by Django 4.2.11 on 2024-05-07 22:25
+# Generated by Django 4.2.14 on 2024-07-30 19:01
 
 from django.db import migrations, models
 import django.db.models.deletion
@@ -7,7 +7,7 @@
 class Migration(migrations.Migration):
 
     dependencies = [
-        ("library", "0026_alter_codebasereleaseplatformtag_tag_and_more"),
+        ("library", "0028_contributor_json_affiliations"),
     ]
 
     operations = [

django/library/models.py

@@ -564,7 +564,7 @@ def latest_remote_release(self):
 
     @property
     def unmirrored_local_releases(self):
-        return self.codebase.ordered_releases().exclude(
+        return self.codebase.public_releases().exclude(
             id__in=self.local_releases.values_list("id", flat=True)
         )
 
@@ -861,20 +861,21 @@ def download_count(self):
             release__codebase__id=self.id
         ).count()
 
-    def ordered_releases(self, has_change_perm=False, asc=True, **kwargs):
+    def ordered_releases_list(self, has_change_perm=False, asc=True, **kwargs):
         """
         list public releases (or all release if has_change_perm is True) in ascending (default) or descending order
         """
         if has_change_perm:
             releases = self.releases.filter(**kwargs)
         else:
-            releases = self.releases.filter(
-                status=CodebaseRelease.Status.PUBLISHED, **kwargs
-            )
+            releases = self.public_releases(**kwargs)
         releases_list = list(releases)
         releases_list.sort(key=lambda r: Version(r.version_number), reverse=not asc)
         return releases_list
 
+    def public_releases(self, **kwargs):
+        return self.releases.filter(status=CodebaseRelease.Status.PUBLISHED, **kwargs)
+
     @classmethod
     def get_list_url(cls):
         return reverse("library:codebase-list")

django/library/tests/base.py

@@ -1,4 +1,5 @@
 import io
+from pathlib import Path
 import random
 from uuid import UUID
 
@@ -15,6 +16,8 @@
 )
 from library.serializers import CodebaseSerializer
 
+TEST_SAMPLES_DIR = Path("library/tests/samples")
+
 
 class CodebaseFactory(ContentModelFactory):
     model = Codebase
@@ -170,7 +173,7 @@ def setUpReviewData(cls):
 
 class ReleaseSetup:
     @classmethod
-    def setUpPublishableDraftRelease(cls, codebase):
+    def setUpPublishableDraftRelease(cls, codebase, with_files=True):
         draft_release = codebase.create_release(
             status=CodebaseRelease.Status.DRAFT,
             initialize=True,
@@ -182,15 +185,14 @@ def setUpPublishableDraftRelease(cls, codebase):
         release_contributor_factory = ReleaseContributorFactory(draft_release)
         contributor = contributor_factory.create()
         release_contributor_factory.create(contributor)
-
-        code_file = io.BytesIO(b"print('hello world')")
-        code_file.name = "some_code_file.py"
-        docs_file = io.BytesIO(b"# Documentation")
-        docs_file.name = "some_doc_file.md"
-        fs_api = draft_release.get_fs_api()
-        fs_api.add(content=code_file, category=FileCategoryDirectories.code)
-        fs_api.add(content=docs_file, category=FileCategoryDirectories.docs)
+        if with_files:
+            code_file = io.BytesIO(b"print('hello world')")
+            code_file.name = "some_code_file.py"
+            docs_file = io.BytesIO(b"# Documentation")
+            docs_file.name = "some_doc_file.md"
+            fs_api = draft_release.get_fs_api()
+            fs_api.add(content=code_file, category=FileCategoryDirectories.code)
+            fs_api.add(content=docs_file, category=FileCategoryDirectories.docs)
 
         draft_release.save()
-
         return draft_release

django/library/tests/samples/releases/animals-model/1.0.0/code/model.py

@@ -0,0 +1 @@
+print("hello world")

django/library/tests/samples/releases/animals-model/1.0.0/data/input.csv

@@ -0,0 +1,2 @@
+horses,sheep
+10,15

django/library/tests/samples/releases/animals-model/1.0.0/docs/README.txt

@@ -0,0 +1,3 @@
+instructions:
+
+python model.py

django/library/tests/samples/releases/animals-model/2.0.0/code/animals/cow.py

@@ -0,0 +1,3 @@
+class Cow:
+    def __init__(self, name):
+        self.name = name

django/library/tests/samples/releases/animals-model/2.0.0/code/animals/horse.py

@@ -0,0 +1,3 @@
+class Horse:
+    def __init__(self, name):
+        self.name = name

django/library/tests/samples/releases/animals-model/2.0.0/code/animals/sheep.py

@@ -0,0 +1,3 @@
+class Sheep:
+    def __init__(self, name):
+        self.name = name

django/library/tests/samples/releases/animals-model/2.0.0/code/model.py

@@ -0,0 +1 @@
+print("hello world")

django/library/tests/samples/releases/animals-model/2.0.0/data/input.csv

@@ -0,0 +1,2 @@
+horses,sheep,cows
+10,15,5

django/library/tests/samples/releases/animals-model/2.0.0/docs/README.md

@@ -0,0 +1,5 @@
+# instructions:
+
+```
+python model.py
+```

django/library/tests/test_fs.py

@@ -1,19 +1,22 @@
 from pathlib import Path
-
+import os
+from git import Repo
 from django.test import TestCase
 
 from core.tests.base import (
     UserFactory,
     destroy_test_shared_folders,
     initialize_test_shared_folders,
+    clear_test_shared_folder,
 )
 from library.fs import (
     FileCategoryDirectories,
     StagingDirectories,
     MessageLevels,
     import_archive,
 )
-from library.tests.base import CodebaseFactory
+from library.tests.base import CodebaseFactory, TEST_SAMPLES_DIR
+from library.models import License
 
 
 import logging
@@ -26,7 +29,7 @@ def setUpModule():
 
 
 class ArchiveExtractorTestCase(TestCase):
-    nested_code_folder = Path("library/tests/archives/nestedcode")
+    nested_code_folder = TEST_SAMPLES_DIR / "archives" / "nestedcode"
 
     def setUp(self):
         self.user_factory = UserFactory()
@@ -70,7 +73,7 @@ def test_zipfile_saving(self):
         )
 
     def test_invalid_zipfile_saving(self):
-        archive_name = "library/tests/archives/invalid.zip"
+        archive_name = str(TEST_SAMPLES_DIR / "archives" / "invalid.zip")
         fs_api = self.codebase_release.get_fs_api()
         with open(archive_name, "rb") as f:
             msgs = fs_api.add(
@@ -86,5 +89,163 @@ def tearDownClass(cls):
         cls.nested_code_folder.with_suffix(".zip").unlink(missing_ok=True)
 
 
+class GitRepoApiTestCase(TestCase):
+    model_dir = TEST_SAMPLES_DIR / "releases" / "animals-model"
+    release_1_dir = model_dir / "1.0.0"
+    release_2_dir = model_dir / "2.0.0"
+
+    def setUp(self):
+        self.user_factory = UserFactory()
+        self.submitter = self.user_factory.create()
+        self.codebase_factory = CodebaseFactory(submitter=self.submitter)
+        self.codebase = self.codebase_factory.create()
+        self.release_1 = self.codebase.create_release()
+        self.git_mirror = self.codebase.create_git_mirror("animals-model")
+
+    def tearDown(self):
+        clear_test_shared_folder(settings.REPOSITORY_ROOT)
+
+    def test_repo_build(self):
+        update_release_from_sample(
+            self.release_1, self.release_1_dir, version_number="1.0.0"
+        )
+        self.release_1.publish()
+        public_release_count = self.codebase.public_releases().count()
+        self.assertEqual(public_release_count, 1)
+        api = self.git_mirror.get_repo_api()
+        api.build()
+        # check that the mirror model is updated and the repo is built
+        self.assertIsNotNone(self.git_mirror.last_local_update)
+        self.assertEqual(self.git_mirror.local_releases.count(), 1)
+        self.assertTrue(os.path.exists(api.repo_dir))
+        # check git stuff
+        repo = Repo(api.repo_dir)
+        self.assertEqual(sum(1 for _ in repo.iter_commits()), public_release_count)
+        self.assertEqual(len(repo.tags), public_release_count)
+        # check contents
+        self.assertTrue(os.path.exists(api.repo_dir / "codemeta.json"))
+        self.assertTrue(os.path.exists(api.repo_dir / "CITATION.cff"))
+        self.assertTrue(os.path.exists(api.repo_dir / "LICENSE"))
+        fs_api = self.release_1.get_fs_api()
+        fs_api.list(StagingDirectories.sip, FileCategoryDirectories.code)
+        for category in ["code", "data", "docs"]:
+            self.assertTrue(
+                api.dirs_equal(
+                    fs_api.sip_contents_dir / category,
+                    api.repo_dir / category,
+                )
+            )
+
+    def test_repo_append_releases(self):
+        update_release_from_sample(
+            self.release_1, self.release_1_dir, version_number="1.0.0"
+        )
+        self.release_1.publish()
+        api = self.git_mirror.get_repo_api()
+        api.build()
+        self.release_2 = self.codebase.create_release()
+        update_release_from_sample(
+            self.release_2, self.release_2_dir, version_number="2.0.0"
+        )
+        self.release_2.publish()
+        api.append_releases()
+
+        self.assertEqual(self.git_mirror.local_releases.count(), 2)
+        # check git stuff
+        repo = Repo(api.repo_dir)
+        public_release_count = self.codebase.public_releases().count()
+        self.assertEqual(sum(1 for _ in repo.iter_commits()), public_release_count)
+        self.assertEqual(len(repo.tags), public_release_count)
+        # check contents
+        self.assertTrue(os.path.exists(api.repo_dir / "codemeta.json"))
+        self.assertTrue(os.path.exists(api.repo_dir / "CITATION.cff"))
+        self.assertTrue(os.path.exists(api.repo_dir / "LICENSE"))
+        fs_api = self.release_2.get_fs_api()
+        fs_api.list(StagingDirectories.sip, FileCategoryDirectories.code)
+        for category in ["code", "data", "docs"]:
+            self.assertTrue(
+                api.dirs_equal(
+                    fs_api.sip_contents_dir / category,
+                    api.repo_dir / category,
+                )
+            )
+
+    def test_will_not_append_lower_version(self):
+        # publish 2.0.0 first and build
+        update_release_from_sample(
+            self.release_1, self.release_1_dir, version_number="1.0.0"
+        )
+        self.release_2 = self.codebase.create_release()
+        update_release_from_sample(
+            self.release_2, self.release_2_dir, version_number="2.0.0"
+        )
+        self.release_2.publish()
+        api = self.git_mirror.get_repo_api()
+        api.build()
+        # now publish release 1.0.0
+        self.release_1.publish()
+        self.assertRaises(ValueError, api.append_releases)
+
+    def test_repo_rebuild(self):
+        update_release_from_sample(
+            self.release_1, self.release_1_dir, version_number="1.0.0"
+        )
+        self.release_1.publish()
+        api = self.git_mirror.get_repo_api()
+        api.build()
+        self.release_2 = self.codebase.create_release()
+        update_release_from_sample(
+            self.release_2, self.release_2_dir, version_number="2.0.0"
+        )
+        self.release_2.publish()
+        api.build()
+
+        self.assertEqual(self.git_mirror.local_releases.count(), 2)
+        # check git stuff
+        repo = Repo(api.repo_dir)
+        public_release_count = self.codebase.public_releases().count()
+        self.assertEqual(sum(1 for _ in repo.iter_commits()), public_release_count)
+        self.assertEqual(len(repo.tags), public_release_count)
+        # check contents
+        self.assertTrue(os.path.exists(api.repo_dir / "codemeta.json"))
+        self.assertTrue(os.path.exists(api.repo_dir / "CITATION.cff"))
+        self.assertTrue(os.path.exists(api.repo_dir / "LICENSE"))
+        fs_api = self.release_2.get_fs_api()
+        fs_api.list(StagingDirectories.sip, FileCategoryDirectories.code)
+        for category in ["code", "data", "docs"]:
+            self.assertTrue(
+                api.dirs_equal(
+                    fs_api.sip_contents_dir / category,
+                    api.repo_dir / category,
+                )
+            )
+
+
 def tearDownModule():
     destroy_test_shared_folders()
+
+
+# helpers ================================================
+
+
+def upload_category(fs_api, release_dir: Path, category: str):
+    category_path = release_dir / category
+    for filepath in category_path.rglob("*"):
+        if filepath.is_file():
+            with filepath.open("rb") as f:
+                relpath = filepath.relative_to(category_path)
+                file_name = str(relpath)
+                fs_api.add(FileCategoryDirectories[category], content=f, name=file_name)
+
+
+def update_release_from_sample(release, sample_dir, version_number):
+    release.os = "Linux"
+    release.programming_languages.add("Python")
+    release.license = License.objects.create(name="MIT")
+    release.release_notes = "Initial release"
+    release.version_number = version_number
+    release.save()
+    fs_api = release.get_fs_api()
+    for category in ["code", "data", "docs"]:
+        upload_category(fs_api, sample_dir, category)
+    return release