Skip to content

Commit

Permalink
Merge branch 'master' into publish_all
Browse files Browse the repository at this point in the history
  • Loading branch information
j-rivero authored Apr 12, 2021
2 parents 9cac6c9 + 7ec026a commit 5f1690d
Show file tree
Hide file tree
Showing 6 changed files with 83 additions and 24 deletions.
28 changes: 26 additions & 2 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,19 @@ on: [push, pull_request]
jobs:
bionic-ci:
runs-on: ubuntu-18.04
name: Bionic Aptly CI -
name: Bionic Aptly ${{ matrix.job_type }} -
debug ${{ matrix.show_debug_cmds }}
strategy:
matrix:
show_debug_cmds:
- true
- false
job_type:
- 'CI'
- 'CONFIG_TESTER'
exclude:
- job_type: 'CONFIG_TESTER'
show_debug_cmds: false
steps:
- name: Checkout
uses: actions/checkout@v2
Expand Down Expand Up @@ -52,9 +58,27 @@ jobs:
%no-protection
EOF
gpg --gen-key --batch genkey
- name: Run tests
# CI TESTING
- name: Run CI tests
if: ${{ matrix.job_type == 'CI' }}
run: |
python3 scripts/aptly/aptly_importer_TEST.py
env:
_DEBUG_MSGS_REPREPRO_UPDATER_TEST_SUITE_: ${{ matrix.show_debug_cmds }}
_ALLOW_DESTRUCTIVE_TESTS_REPREPRO_UPDATER_TEST_SUITE_: true
# CONFIG FILE TESTER
- id: files
if: ${{ matrix.job_type == 'CONFIG_TESTER' }}
uses: jitterbit/get-changed-files@v1
- name: Run testing on changed config files
if: ${{ matrix.job_type == 'CONFIG_TESTER' }}
run: |
for changed_file in ${{ steps.files.outputs.all }}; do
if [[ ${changed_file} != ${changed_file/config\/*.yaml} ]]; then
echo "+ Detected config file: ${changed_file}."
python3 scripts/aptly/aptly_importer.py --ignore-signatures --only-mirror-creation ${changed_file}
else
echo "- Ignoring non config file ${changed_file}."
fi
done
1 change: 1 addition & 0 deletions config/libopensplice67_bionic_xenial.yaml
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
---
name: libopensplice67_bionic_xenial
method: http://packages.osrfoundation.org/gazebo/ubuntu-prerelease
suites: [bionic, xenial]
Expand Down
1 change: 1 addition & 0 deletions config/opensplice69.osrfoundation.yaml
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
---
name: opensplice69
method: http://packages.osrfoundation.org/gazebo/ubuntu-stable
suites: [xenial, bionic]
Expand Down
5 changes: 3 additions & 2 deletions config/pyside2.prerelease.yaml
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
---
name: pyside2
method: http://ppa.launchpad.net/tully.foote/pyside2-reproduction/ubuntu/
suites: [xenial]
Expand Down Expand Up @@ -46,9 +47,9 @@ Package (= python-pyside2.qtwidgets) |\
Package (= python-pyside2.qtwidgets-dbg) |\
Package (= python-pyside2.qtx11extras) |\
Package (= python-pyside2.qtx11extras-dbg) |\
Package (= python-pyside2.qtxml) \
Package (= python-pyside2.qtxml) |\
Package (= python-pyside2.qtxml-dbg)), \
$Version (% 2.0.0+dev1-1~202102130210~rev1858~pkg5~git131fdfd1~ubuntu16.04.1) |\
$Version (% 2.0.0+dev1-1~202102130210~rev1858~pkg5~git131fdfd1~ubuntu16.04.1)) |\
((Package (= shiboken2) |\
Package (= libshiboken-py3-2.0) |\
Package (= libshiboken2-dev) |\
Expand Down
57 changes: 43 additions & 14 deletions scripts/aptly/aptly_importer.py
100644 → 100755
Original file line number Diff line number Diff line change
Expand Up @@ -183,7 +183,7 @@ def __error(self, msg):
exit(2)

def __load_config_file(self, config_file_path):
fn = Path(__file__).parent / config_file_path
fn = Path(config_file_path).absolute()
try:
with open(str(fn), 'r') as stream:
config = yaml.safe_load(stream)
Expand All @@ -195,11 +195,18 @@ def __load_config_file(self, config_file_path):


class UpdaterManager():
def __init__(self, input_file, debug=False, aptly_config_file=None, simulate_repo_import=False, snapshot_and_publish=False):
def __init__(self, input_file, debug=False,
aptly_config_file=None,
ignore_mirror_signature=False,
only_mirror_creation=False,
simulate_repo_import=False,
snapshot_and_publish=False):
self.aptly = Aptly(debug,
config_file=aptly_config_file)
self.config = UpdaterConfiguration(input_file)
self.debug = debug
self.ignore_mirror_signature = ignore_mirror_signature
self.only_mirror_creation = only_mirror_creation
self.simulate_repo_import = simulate_repo_import
self.snapshot_and_publish = snapshot_and_publish
self.snapshot_timestamp = None
Expand All @@ -211,14 +218,22 @@ def __create_aptly_mirror(self, distribution):
if self.aptly.exists(Aptly.ArtifactType.MIRROR, mirror_name):
self.__log_ok('Removing existing mirror')
self.aptly.run(['mirror', 'drop', mirror_name])
self.aptly.run(['mirror', 'create', '-with-sources',
f"-architectures={','.join(self.config.architectures)}",
f"-filter={self.config.filter_formula}",
mirror_name,
self.config.method,
distribution,
self.config.component])
self.aptly.run(['mirror', 'update', mirror_name])
create_cmd = ['mirror', 'create', '-with-sources',
f"-architectures={','.join(self.config.architectures)}",
f"-filter={self.config.filter_formula}"]
if self.ignore_mirror_signature:
create_cmd += ['-ignore-signatures']
create_cmd += [mirror_name,
self.config.method,
distribution,
self.config.component]
self.aptly.run(create_cmd)
update_cmd = ['mirror', 'update']
if self.ignore_mirror_signature:
update_cmd += ['-ignore-signatures']
update_cmd += [mirror_name]
self.aptly.run(update_cmd)

self.__log_ok(f"mirror {mirror_name} created")

def __create_aptly_snapshot_from_repo(self, distribution):
Expand Down Expand Up @@ -331,7 +346,9 @@ def run(self):
self.__get_mirror_name(dist)):
self.__remove_all_generated_mirrors()
self.__error(f'{self.__get_mirror_name(dist)} does not have a source package. Removing generated mirrors')
self.__log_ok('There is a source pakage in the mirror')
self.__log_ok('All source packages exist in the mirror')
if self.only_mirror_creation:
return True
# 2. Import from mirrors to local repositories
self.__import__aptly_mirror_to_repo(dist)
if self.simulate_repo_import:
Expand All @@ -351,11 +368,19 @@ def run(self):

def main():
"""
Usage: python3 aptly_importer.py [--simulate-repo-import] <config_file>
Usage: python3 aptly_importer.py [parameters] <config_file>
"""
usage = "usage: %prog [--simulate-repo-import] config_file"
usage = "usage: python3 aptly_importer.py [parameters] <config_file>"
parser = argparse.ArgumentParser(usage)
parser.add_argument('config_file', type=str, default=None)
parser.add_argument("--ignore-signatures",
help="Ignore mirror signatures when importing",
action="store_true")
parser.add_argument("--only-mirror-creation",
help="Perform only the mirror creation and update. Useful for"
"checking, it does not modify anything in the system",
action="store_true")

parser.add_argument('--simulate-repo-import',
action='store_true',
help='Perform a dry-run until the point of simulation mirrors import to repositories')
Expand All @@ -366,7 +391,11 @@ def main():
if not path.exists(args.config_file):
parser.error("Missing input file from %s" % args.config_file)

manager = UpdaterManager(args.config_file, simulate_repo_import=args.simulate_repo_import, snapshot_and_publish=args.snapshot_and_publish)
manager = UpdaterManager(input_file=args.config_file,
ignore_mirror_signature=args.ignore_signatures,
only_mirror_creation=args.only_mirror_creation,
simulate_repo_import=args.simulate_repo_import,
snapshot_and_publish=args.snapshot_and_publish)
manager.run()


Expand Down
15 changes: 9 additions & 6 deletions scripts/aptly/aptly_importer_TEST.py
Original file line number Diff line number Diff line change
@@ -1,10 +1,13 @@
import aptly_importer
import os
from pathlib import Path
import sys
import tempfile
import unittest


TEST_CONFIG_DIR = Path(__file__).parent.absolute() / 'test'

class TestYamlConfiguration(unittest.TestCase):
def test_non_existsing_file(self):
with self.assertRaises(SystemExit):
Expand All @@ -15,7 +18,7 @@ def test_missing_field(self):
aptly_importer.UpdaterConfiguration('test/missing_architectures_field.yaml')

def test_ok(self):
self.assertTrue(aptly_importer.UpdaterConfiguration('test/example.yaml'))
self.assertTrue(aptly_importer.UpdaterConfiguration(TEST_CONFIG_DIR / 'example.yaml'))


class TestAptly(unittest.TestCase):
Expand Down Expand Up @@ -126,7 +129,7 @@ def __setup__(self, distros_expected, config_file, snapshot_and_publish=True):
for distro in self.expected_distros]
self.expected_repos_by_distro_test_name =\
{f"{distro}": f"ros_bootstrap-{distro}" for distro in self.expected_distros}
self.manager = aptly_importer.UpdaterManager(config_file,
self.manager = aptly_importer.UpdaterManager(TEST_CONFIG_DIR / config_file,
debug=self.debug_msgs,
aptly_config_file=self.aptly_config_file,
snapshot_and_publish=snapshot_and_publish)
Expand All @@ -139,24 +142,24 @@ def test_no_publish_basic_example_creation_from_scratch(self):
self.__assert_expected_repos_mirrors()

def test_basic_example_creation_from_scratch(self):
self.__setup__(['focal', 'groovy'], 'test/example.yaml')
self.__setup__(['focal', 'groovy'], 'example.yaml')
self.assertTrue(self.manager.run())
self.__assert_expected_repos_mirrors()

def test_basic_example_creation_existing_mirror(self):
self.__setup__(['focal', 'groovy'], 'test/example.yaml')
self.__setup__(['focal', 'groovy'], 'example.yaml')
[self.__add_mirror(name) for name in self.expected_mirrors_test_name]
self.assertTrue(self.manager.run())
self.__assert_expected_repos_mirrors()

def test_basic_example_creation_existing_repo(self):
self.__setup__(['focal', 'groovy'], 'test/example.yaml')
self.__setup__(['focal', 'groovy'], 'example.yaml')
[self.__add_repo(name) for name in self.expected_repos_test_name]
self.assertTrue(self.manager.run())
self.__assert_expected_repos_mirrors()

def test_example_no_sources(self):
self.__setup__(['xenial'], 'test/example_no_source_package.yaml')
self.__setup__(['xenial'], 'example_no_source_package.yaml')
with self.assertRaises(SystemExit):
self.manager.run()
self.__assert_no_mirrors()
Expand Down

0 comments on commit 5f1690d

Please sign in to comment.