Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Change aptly_importer behaviour to publish all existing repositories #108

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Change aptly_importer behaviour to publish all existing repositories #108

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 39 additions & 9 deletions scripts/aptly/aptly_importer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -236,24 +236,48 @@ def __create_aptly_mirror(self, distribution):

self.__log_ok(f"mirror {mirror_name} created")

def __create_aptly_snapshot(self, distribution):
self.__log('Creating an aptly snapshot from local aptly repository')
def __create_aptly_snapshot_from_repo(self, distribution):
self.aptly.run(['snapshot', 'create', self.__get_snapshot_name(distribution),
'from', 'repo', self.__get_repo_name(distribution)])
self.__log_ok(f"snapshot {self.__get_snapshot_name(distribution)} created from repo {self.__get_repo_name(distribution)}")

def __create_aptly_snapshot_from_all_repos(self):
for repo in self.__get_all_repos():
self.__create_aptly_snapshot_from_repo(self.__get_distro_from_repo_name(repo))

def __error(self, msg):
print(f"Update Manager error: {msg} \n", file=stderr)
exit(1)

def __get_all_repos(self):
cmd = ['repo', 'list']
run_result = self.aptly.run(cmd, return_all_info=True)
if run_result.returncode != 0:
self.__error(run_result.stderr.decode('utf-8'))

all_repos = []
for line in run_result.stdout.splitlines():
if f"[{self.__get_repo_name_prefix()}-" in line.decode():
m = re.findall(r"\[(.*)\] \(packages", line.decode())
all_repos.append(m[0])
return all_repos

def __get_distro_from_repo_name(self, repo_name):
distro_regexp = re.findall(r'%s-(.*)' % self.__get_repo_name_prefix(), repo_name)
assert(len(distro_regexp) == 1)
return distro_regexp[0]

def __get_endpoint_name(self, distribution):
return f"filesystem:live:ros_bootstrap"

def __get_mirror_name(self, distribution):
return f"{self.config.name}-{distribution}"

def __get_repo_name(self, distribution):
return f"ros_bootstrap-{distribution}"
return f"{self.__get_repo_name_prefix()}-{distribution}"

def __get_repo_name_prefix(self):
return 'ros_bootstrap'

def __get_snapshot_name(self, distribution):
if not self.snapshot_timestamp:
Expand Down Expand Up @@ -286,8 +310,11 @@ def __log(self, msg):
def __log_ok(self, msg):
self.__log(f" [ok] {msg}")

def __publish_all_new_snapshot(self):
for repo in self.__get_all_repos():
self.__publish_new_snapshot(self.__get_distro_from_repo_name(repo))

def __publish_new_snapshot(self, dist):
self.__log('Publish the new snapshot')
if (self.aptly.exists_publication(dist, self.__get_endpoint_name(dist))):
self.aptly.run(['publish', 'switch',
dist,
Expand Down Expand Up @@ -327,11 +354,14 @@ def run(self):
if self.simulate_repo_import:
self.__log_ok(f"Simulation of the import actions from mirrors to repos finished")
exit(0)
if self.snapshot_and_publish:
# 3. Create snapshots from repositories
self.__create_aptly_snapshot(dist)
# 4. Publish new snapshots
self.__publish_new_snapshot(dist)

if self.snapshot_and_publish:
# 3. Create snapshots from all existing repositories
self.__log('Creating snapshots from all repositories')
self.__create_aptly_snapshot_from_all_repos()
# 4. Publish new snapshots
self.__log('Publishing all new snapshots')
self.__publish_all_new_snapshot()
self.__log(f"\n == [ END OF PROCESSING {self.config.name} ] ==\n")
return True

Expand Down
36 changes: 33 additions & 3 deletions scripts/aptly/aptly_importer_TEST.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,11 @@ def __add_mirror(self, mirror_name):
self.assertTrue(self.aptly.run(['mirror', 'create', mirror_name, 'http://packages.osrfoundation.org/gazebo/ubuntu-stable', 'focal']))

def __add_repo(self, repo_name):
self.assertTrue(self.aptly.run(['repo', 'create', repo_name]))
self.assertTrue(self.aptly.run([
'repo',
'create',
f"-architectures={','.join(self.manager.config.architectures)}",
repo_name]))

def __assert_no_mirrors(self):
for name in self.expected_mirrors_test_name:
Expand All @@ -92,6 +96,8 @@ def __clean_up_aptly_test_artifacts(self):
[self.__remove_repo(name) for name in self.expected_repos_test_name]
[self.__remove_mirror(name) for name in self.expected_mirrors_test_name]
[self.__remove_snapshots_from_mirror(name) for name in self.expected_mirrors_test_name]
# extra artifacts in some tests
self.__remove_repo('ros_bootstrap-xenial')
self.aptly.run(['db', 'cleanup'])

def __remove_mirror(self, mirror_name):
Expand All @@ -114,7 +120,7 @@ def __remove_snapshots_from_mirror(self, mirror_name):
for snap in self.aptly.get_snapshots_from_mirror(mirror_name):
self.aptly.run(['snapshot', 'drop', snap])

def __setup__(self, distros_expected, config_file):
def __setup__(self, distros_expected, config_file, snapshot_and_publish=True):
self.expected_distros = distros_expected
self.expected_endpoint_name = 'filesystem:live:ros_bootstrap'
self.expected_mirrors_test_name = [f"_reprepro_updater_test_suite_-{distro}"
Expand All @@ -126,10 +132,15 @@ def __setup__(self, distros_expected, config_file):
self.manager = aptly_importer.UpdaterManager(TEST_CONFIG_DIR / config_file,
debug=self.debug_msgs,
aptly_config_file=self.aptly_config_file,
snapshot_and_publish=True)
snapshot_and_publish=snapshot_and_publish)
# clean up testing artifacts if they previously exists
self.__clean_up_aptly_test_artifacts()

def test_no_publish_basic_example_creation_from_scratch(self):
self.__setup__(['focal', 'groovy'], 'test/example.yaml', snapshot_and_publish=False)
self.assertTrue(self.manager.run())
self.__assert_expected_repos_mirrors()

def test_basic_example_creation_from_scratch(self):
self.__setup__(['focal', 'groovy'], 'example.yaml')
self.assertTrue(self.manager.run())
Expand All @@ -153,6 +164,25 @@ def test_example_no_sources(self):
self.manager.run()
self.__assert_no_mirrors()

def test_publish_existing_repos_in_config(self):
self.__setup__(['focal', 'groovy'], 'test/example.yaml')
# add existing repositories
self.__add_repo('ros_bootstrap-focal')
self.assertTrue(self.manager.run())
self.__assert_expected_repos_mirrors()

def test_publish_existing_repos_not_in_config(self):
self.__setup__(['focal', 'groovy'], 'test/example.yaml')
# add existing repositories
self.__add_repo('ros_bootstrap-xenial')
[self.__add_repo(name) for name in self.expected_repos_test_name]
# publishing neeeds custom architectures parameter to work nicely on created xenial
# repo. We expect failure to indicate the xenial is in the mix which is the expected
# behaviour. To solve the problem extra support needs to be added to aptly_imported
# to declare architectures
with self.assertRaises(SystemExit):
self.assertTrue(self.manager.run())


class TestReprepro2AptlyFilter(unittest.TestCase):
def setUp(self):
Expand Down