From 5f45009a31f7793d59fbad0b0546a217f2800994 Mon Sep 17 00:00:00 2001 From: Manisha Singhal Date: Thu, 28 Oct 2021 09:35:15 +0200 Subject: [PATCH] Closes #396 --- CHANGES/396.feature | 1 + MANIFEST.in | 1 + pulp_deb/app/depsolving.py | 432 ++++++++++++++++++ pulp_deb/app/schema/__init__.py | 7 + pulp_deb/app/schema/copy_config.json | 21 + pulp_deb/app/serializers/__init__.py | 2 +- .../app/serializers/repository_serializers.py | 46 +- pulp_deb/app/tasks/__init__.py | 1 + pulp_deb/app/tasks/copy.py | 116 +++++ pulp_deb/app/urls.py | 5 + pulp_deb/app/viewsets/__init__.py | 2 +- pulp_deb/app/viewsets/repository.py | 89 +++- requirements.txt | 1 + 13 files changed, 716 insertions(+), 8 deletions(-) create mode 100644 CHANGES/396.feature create mode 100644 pulp_deb/app/depsolving.py create mode 100644 pulp_deb/app/schema/__init__.py create mode 100644 pulp_deb/app/schema/copy_config.json create mode 100644 pulp_deb/app/tasks/copy.py create mode 100644 pulp_deb/app/urls.py diff --git a/CHANGES/396.feature b/CHANGES/396.feature new file mode 100644 index 000000000..6e1a04e13 --- /dev/null +++ b/CHANGES/396.feature @@ -0,0 +1 @@ +Add advanced copy workflows to be used by Katello diff --git a/MANIFEST.in b/MANIFEST.in index 4e1ac30b2..26017f666 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,6 +3,7 @@ include COMMITMENT include COPYRIGHT include functest_requirements.txt include LICENSE +include pulp_deb/app/schema/* include pulp_deb/tests/functional/sign_deb_release.sh include pyproject.toml include requirements.txt diff --git a/pulp_deb/app/depsolving.py b/pulp_deb/app/depsolving.py new file mode 100644 index 000000000..93ec19c9f --- /dev/null +++ b/pulp_deb/app/depsolving.py @@ -0,0 +1,432 @@ +import collections +import logging +import solv +import re + +from pulp_deb.app import models + + +logger = logging.getLogger(__name__) + +# The name for the repo inside libsolv which represents the combined set of target/destination +# repositories. Libsolv only supports one "installed" repo at a time, therefore we need to +# combine them and determine what units actually go where afterwards. +COMBINED_TARGET_REPO_NAME = "combined_target_repo" + +# Constants for loading data from the database. +DEB_FIELDS = ["pk", "package", "version", "architecture", "provides", "depends"] + + +def deb_to_solvable(solv_repo, unit): + """Convert a Pulp DEB dict to a libsolv solvable. + + Args: + solv_repo (solv.Repo): The libsolv repository the unit is being created in. + unit (dict): The unit being converted. + + Returns: + (solv.Solvable) The solvable created. + + """ + solvable = solv_repo.add_solvable() + + def deb_filelist_conversion(solvable, unit): + """A specific, deb-unit-type filelist attribute conversion.""" + repodata = solv_repo.first_repodata() + + for file_repr in unit.get("files", []): + # file_repr = e.g. (None, '/usr/bin/', 'bash') + file_dir = file_repr[1] + file_name = file_repr[2] + if not file_dir: + # https://github.com/openSUSE/libsolv/issues/397 + continue + dirname_id = repodata.str2dir(file_dir) + repodata.add_dirstr(solvable.id, solv.SOLVABLE_FILELIST, dirname_id, file_name) + + def deb_basic_deps(solvable, name, version, arch): + # Prv: $n . $a = $evr + pool = solvable.repo.pool + name_id = pool.str2id(name) + version_id = pool.str2id(version) + arch_id = pool.str2id(arch) + rel = pool.rel2id(name_id, arch_id, solv.REL_ARCH) + rel = pool.rel2id(rel, version_id, solv.REL_EQ) + solvable.add_deparray(solv.SOLVABLE_PROVIDES, rel) + + name = unit.get("package") + solvable.name = name + + version = unit.get("version") + solvable.version = version + + arch = unit.get("architecture", "noarch") + solvable.arch = arch + + for depunit in unit.get("depends").split(", "): + deb_dependency_conversion(solvable, re.split(" ", depunit)) + + deb_filelist_conversion(solvable, unit) + deb_basic_deps(solvable, name, version, arch) + + return solvable + + +def deb_dependency_conversion(solvable, unit, dependency_key=None): + """Set the solvable dependencies. + + The dependencies of a unit are stored as a list of dictionaries, + containing following values: + name: or a rich dep string; mandatory + version: version of the dependency; optional + flags: AND/OR; optional; if missing meaning by default AND + + These values are parsed by libdeb. + There are two cases how libsolv addresses the dependencies: + + * rich: the name of the dependency contains all required information: + '(foo >= 1.0-3 AND bar != 0.9)' + all the other attribute values are ignored + + * generic: the name, version and flags attributes + are processed explicitly + + The dependency list is either of the provides, requires or the weak + dependencies, the current case being stored under self.attr_name. + + Libsolv tracks a custom Dep object to represent a dependency of a + solvable object; these are created in the pool object: + + dependency = pool.Dep('foo') + + The relationship to the solvable is tracked by a Rel pool object: + + relationship = pool.Rel(solv.REL_AND, pool.Dep(version)) + + The relationship is then recorded on the solvable explicitly by: + + solvable.add_deparray(solv.SOLVABLE_PROVIDES, relationship) + + If no explict relationship is provided in the flags attribute, + the dependency can be used directly: + + solvable.add_deparray(solv.SOLVABLE_PROVIDES, dependency) + + Args: + solvable (solvable): a libsolv solvable object + unit (dict): the content unit to get the dependencies from + + """ + unit_name = unit[0] + unit_flags = None + unit_version = None + if len(unit) > 1: + unit_flags = unit[1][1:] + unit_version = unit[2][:-1] + + # e.g SOLVABLE_PROVIDES, SOLVABLE_REQUIRES... + keyname = dependency_key or getattr(solv, "SOLVABLE_{}".format("REQUIRES")) + pool = solvable.repo.pool + # generic dependencies provide at least a solvable name + dep = pool.Dep(unit_name) + if unit_flags: + # in case the flags unit attribute is populated, use it as + # a solv.Rel object to denote solvable--dependency + # relationship dependency in this case is a relationship + # towards the dependency made from the 'flags', e.g: + # solv.REL_EQ, and the evr fields + if unit_flags == "=": + rel_flags = solv.REL_EQ + elif unit_flags == "<": + rel_flags = solv.REL_LT + elif unit_flags == ">": + rel_flags = solv.REL_GT + elif unit_flags == "<=": + rel_flags = solv.REL_EQ | solv.REL_LT + elif unit_flags == ">=": + rel_flags = solv.REL_EQ | solv.REL_GT + else: + raise ValueError("Unsupported dependency flags %s" % unit_flags) + dep = dep.Rel(rel_flags, pool.Dep(unit_version)) + # register the constructed solvable dependency + solvable.add_deparray(keyname, dep) + + +class UnitSolvableMapping: + """Map libsolv solvables to Pulp units and repositories. + + Translate between what libsolv understands, solvable IDs in a pool, and what Pulp understands, + units and repositories. + """ + + def __init__(self): + """Mapping Init.""" + # Stores data in the form (pulp_unit_id, pulp_repo_id): solvable + self._mapping_unit_to_solvable = {} + # Stores data in the form solvable_id: (pulp_unit_id, pulp_repo_id) + self._mapping_solvable_to_unit = {} + # Stores data in the form pulp_repo_id: libsolv_repo_id + self._mapping_repos = {} + + def register(self, unit_id, solvable, repo_id): + """Store the matching of a unit-repo pair to a solvable inside of the mapping.""" + if not self.get_repo(str(repo_id)): + raise ValueError( + "Attempting to register unit {} to unregistered repo {}".format(unit_id, repo_id) + ) + + self._mapping_solvable_to_unit.setdefault(solvable.id, (unit_id, repo_id)) + self._mapping_unit_to_solvable.setdefault((unit_id, repo_id), solvable) + + logger.debug( + "Loaded unit {unit}, {repo} as {solvable}".format( + unit=unit_id, solvable=solvable, repo=repo_id + ) + ) + + def register_repo(self, repo_id, libsolv_repo): + """Store the repo (Pulp) - repo (libsolv) pair.""" + return self._mapping_repos.setdefault(str(repo_id), libsolv_repo) + + def get_repo(self, repo_id): + """Return the repo from the mapping.""" + return self._mapping_repos.get(repo_id) + + def get_unit_id(self, solvable): + """Get the (unit, repo_id) pair for a given solvable.""" + return self._mapping_solvable_to_unit.get(solvable.id) + + def get_solvable(self, unit_id, repo_id): + """Fetch the libsolv solvable associated with a unit-repo pair.""" + return self._mapping_unit_to_solvable.get((unit_id, repo_id)) + + def get_repo_units(self, repo_id): + """Get back unit ids of all units that were in a repo based on the mapping.""" + return set( + unit_id + for (unit_id, unit_repo_id) in self._mapping_unit_to_solvable.keys() + if unit_repo_id == repo_id + ) + + def get_units_from_solvables(self, solvables): + """Map a list of solvables into their Pulp units, keyed by the repo they came from.""" + repo_unit_map = collections.defaultdict(set) + for solvable in solvables: + (unit_id, repo_id) = self.get_unit_id(solvable) + repo_unit_map[repo_id].add(unit_id) + + return repo_unit_map + + +class Solver: + """A Solver object that can speak in terms of Pulp units.""" + + def __init__(self): + """Solver Init.""" + self._finalized = False + self._pool = solv.Pool() + self._pool.setarch() # prevent https://github.com/openSUSE/libsolv/issues/267 + self._pool.set_flag(solv.Pool.POOL_FLAG_IMPLICITOBSOLETEUSESCOLORS, 1) + self.mapping = UnitSolvableMapping() + + def finalize(self): + """Finalize the solver - a finalized solver is ready for depsolving. + + Libsolv needs to perform some scans/hashing operations before it can do certain things. + For more details see: + https://github.com/openSUSE/libsolv/blob/master/doc/libsolv-bindings.txt + """ + self._pool.installed = self.mapping.get_repo(COMBINED_TARGET_REPO_NAME) + self._pool.addfileprovides() + self._pool.createwhatprovides() + self._finalized = True + + def load_source_repo(self, repo_version): + """Load the provided Pulp repo as a source repo. + + All units in the repo will be available to be "installed", or copied. + """ + libsolv_repo_name = self._load_from_version(repo_version) + logger.debug( + "Loaded repository '{}' version '{}' as source repo".format( + repo_version.repository, repo_version.number + ) + ) + return libsolv_repo_name + + def load_target_repo(self, repo_version): + """Load the provided Pulp repo into the combined target repo. + + All units in the repo will be added to the combined target repo, the contents of which + are considered "installed" by the solver. + """ + libsolv_repo_name = self._load_from_version(repo_version, as_target=True) + logger.debug( + "Loaded repository '{}' version '{}' into combined target repo".format( + repo_version.repository, repo_version.number + ) + ) + return libsolv_repo_name + + def _repo_version_to_libsolv_name(self, repo_version): + """Produce a name to use for the libsolv repo from the repo version.""" + return "{}: version={}".format(repo_version.repository.name, repo_version.number) + + def _load_from_version(self, repo_version, as_target=False): + """ + Generate solvables from Pulp units and add them to the mapping. + + In some circumstances, we want to load multiple Pulp "repos" together into one libsolv + "repo", because libsolv can only have one repo be "installed" at a time. Therefore, when + the override_repo_name is specified, the created solvables are associated with the + override repo, but the mapping stores them with their original Pulp repo_id. + + Args: + override_repo_name (str): Override name to use when adding solvables to a libsolv repo + """ + if as_target: + libsolv_repo_name = COMBINED_TARGET_REPO_NAME + else: + libsolv_repo_name = self._repo_version_to_libsolv_name(repo_version) + + repo = self.mapping.get_repo(libsolv_repo_name) + if not repo: + repo = self.mapping.register_repo( + libsolv_repo_name, self._pool.add_repo(libsolv_repo_name) + ) + repodata = repo.add_repodata() + else: + repodata = repo.first_repodata() + + # Load packages into the solver + + package_ids = repo_version.content.filter(pulp_type=models.Package.get_pulp_type()).only( + "pk" + ) + + debs = models.Package.objects.filter(pk__in=package_ids).values(*DEB_FIELDS) + + for deb in debs.iterator(chunk_size=5000): + self._add_unit_to_solver(deb_to_solvable, deb, repo, libsolv_repo_name) + + # Need to call pool->addfileprovides(), pool->createwhatprovides() after loading new repo + self._finalized = False + + repodata.internalize() + return libsolv_repo_name + + def _add_unit_to_solver(self, conversion_func, unit, repo, libsolv_repo_name): + solvable = conversion_func(repo, unit) + self.mapping.register(unit["pk"], solvable, libsolv_repo_name) + + def _build_warnings(self, problems): + """Builds a list of 'warnable' depsolving errors. + + "install" problems aren't relevant to "is the repo-version resulting from this copy + dependency-complete", so we choose to ignore them, as they only hide 'real' + depsolving issues while alarming the user. + + Args: + problems: (list) List of libsolv.Problem from latest depsolving attempt + Returns: (list) List of error-strings for 'warnable' problems. + """ + warn_on = [ + solv.Solver.SOLVER_RULE_JOB_NOTHING_PROVIDES_DEP, + solv.Solver.SOLVER_RULE_JOB_UNKNOWN_PACKAGE, + solv.Solver.SOLVER_RULE_PKG, + ] + warnings = [] + for problem in problems: + for problem_rule in problem.findallproblemrules(): + for info in problem_rule.allinfos(): + if info.type in warn_on: + warnings.append(str(info)) + return warnings + + def resolve_dependencies(self, unit_repo_map): + """Resolve the total set of packages needed for the packages passed in, as DNF would. + + Find the set of dependent units and return them in a dictionary where + the key is the repository the set of units came from. + + Find the DEB dependencies that need to be copied to satisfy copying the provided units, + taking into consideration what units are already present in the target repository. + + Create libsolv jobs to install each one of the units passed in, collect and combine the + results. For modules, libsolv jobs are created to install each of their artifacts + separately. + + A libsolv "Job" is a request for the libsolv sat solver to process. For instance a job with + the flags SOLVER_INSTALL | SOLVER_SOLVABLE will tell libsolv to solve an installation of + a package which is specified by solvable ID, as opposed to by name, or by pattern, which + are other options available. + + See: https://github.com/openSUSE/libsolv/blob/master/doc/libsolv-bindings.txt#the-job-class + + In the context of this use of libsolv, we have added/are taking advantage of the flag + solv.Pool.POOL_FLAG_IMPLICITOBSOLETEUSESCOLORS. to handle multiarch repos correctly. + + Args: + unit_repo_map: (dict) An iterable oflibsolv_repo_name = + + Returns: (dict) A dictionary of form {'repo_id': set(unit_ids**)} + """ + assert self._finalized, "Depsolver must be finalized before it can be used" + + solvables = [] + # units (like advisories) that aren't solved for in the dependency solver need to be + # passed through to the end somehow, so let's add them to a second mapping that mirrors + # the first and combine them again at the end. + passthrough = {k: set() for k in unit_repo_map.keys()} + for repo, units in unit_repo_map.items(): + for unit in units: + if unit.pulp_type in {"deb.package"}: + solvables.append(self.mapping.get_solvable(unit.pk, repo)) + passthrough[repo].add(unit.pk) + + self._pool.createwhatprovides() + flags = solv.Job.SOLVER_INSTALL | solv.Job.SOLVER_SOLVABLE + + def run_solver_jobs(jobs): + """Execute the libsolv jobs, return results. + + Take a list of jobs, get a solution, return the set of solvables that needed to + be installed. + """ + solver = self._pool.Solver() + raw_problems = solver.solve(jobs) + # The solver is simply ignoring the problems encountered and proceeds associating + # any new solvables/units. This might be reported back to the user one day over + # the REST API. For now, log only "real" dependency issues (typically some variant + # of "can't find the package" + dependency_warnings = self._build_warnings(raw_problems) + if dependency_warnings: + logger.warning( + "Encountered problems solving dependencies, " + "copy may be incomplete: {}".format(", ".join(dependency_warnings)) + ) + + transaction = solver.transaction() + return set(transaction.newsolvables()) + + solvables_to_copy = set(solvables) + result_solvables = set() + install_jobs = [] + + while solvables_to_copy: + # Take one solvable + solvable = solvables_to_copy.pop() + + # If the unit being copied is not a module, just install it alone + unit_install_job = self._pool.Job(flags, solvable.id) + install_jobs.append(unit_install_job) + + # Depsolve using the list of unit install jobs, add them to the results + solvables_copied = run_solver_jobs(install_jobs) + result_solvables.update(solvables_copied) + + solved_units = self.mapping.get_units_from_solvables(result_solvables) + for k in unit_repo_map.keys(): + solved_units[k] |= passthrough[k] + + return solved_units diff --git a/pulp_deb/app/schema/__init__.py b/pulp_deb/app/schema/__init__.py new file mode 100644 index 000000000..8bb3489c4 --- /dev/null +++ b/pulp_deb/app/schema/__init__.py @@ -0,0 +1,7 @@ +import json +import os + +location = os.path.dirname(os.path.realpath(__file__)) + +with open(os.path.join(location, "copy_config.json")) as copy_config_json: + COPY_CONFIG_SCHEMA = json.load(copy_config_json) diff --git a/pulp_deb/app/schema/copy_config.json b/pulp_deb/app/schema/copy_config.json new file mode 100644 index 000000000..898bcfeda --- /dev/null +++ b/pulp_deb/app/schema/copy_config.json @@ -0,0 +1,21 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "CopyConfig", + "description": "Config for copying content between repos", + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "additionProperties": false, + "required": [ "source_repo_version", "dest_repo" ], + "properties": { + "source_repo_version": { "type": "string" }, + "dest_repo": { "type": "string" }, + "dest_base_version": { "type": "integer" }, + "content": { + "type": "array", + "items": { "type": "string" } + } + } + } +} diff --git a/pulp_deb/app/serializers/__init__.py b/pulp_deb/app/serializers/__init__.py index b6169fbc4..b792264f3 100644 --- a/pulp_deb/app/serializers/__init__.py +++ b/pulp_deb/app/serializers/__init__.py @@ -23,4 +23,4 @@ from .remote_serializers import AptRemoteSerializer -from .repository_serializers import AptRepositorySerializer +from .repository_serializers import AptRepositorySerializer, CopySerializer diff --git a/pulp_deb/app/serializers/repository_serializers.py b/pulp_deb/app/serializers/repository_serializers.py index 8f821c7bd..6cf05da2a 100644 --- a/pulp_deb/app/serializers/repository_serializers.py +++ b/pulp_deb/app/serializers/repository_serializers.py @@ -1,7 +1,12 @@ -from pulpcore.plugin.serializers import RepositorySerializer +from gettext import gettext as _ +from pulpcore.plugin.serializers import RepositorySerializer, validate_unknown_fields from pulp_deb.app.models import AptRepository +from jsonschema import Draft7Validator +from rest_framework import serializers +from pulp_deb.app.schema import COPY_CONFIG_SCHEMA + class AptRepositorySerializer(RepositorySerializer): """ @@ -11,3 +16,42 @@ class AptRepositorySerializer(RepositorySerializer): class Meta: fields = RepositorySerializer.Meta.fields model = AptRepository + + +class CopySerializer(serializers.Serializer): + """ + A serializer for Content Copy API. + """ + + config = serializers.JSONField( + help_text=_("A JSON document describing sources, destinations, and content to be copied") + ) + + dependency_solving = serializers.BooleanField( + help_text=_("Also copy dependencies of the content being copied."), default=False + ) + + def validate(self, data): + """ + Validate that the Serializer contains valid data. + Set the AptRepository based on the RepositoryVersion if only the latter is provided. + Set the RepositoryVersion based on the AptRepository if only the latter is provided. + Convert the human-friendly names of the content types into what Pulp needs to query on. + """ + super().validate(data) + + if hasattr(self, "initial_data"): + validate_unknown_fields(self.initial_data, self.fields) + + if "config" in data: + validator = Draft7Validator(COPY_CONFIG_SCHEMA) + + err = [] + for error in sorted(validator.iter_errors(data["config"]), key=str): + err.append(error.message) + if err: + raise serializers.ValidationError( + _("Provided copy criteria is invalid:'{}'".format(err)) + ) + + return data diff --git a/pulp_deb/app/tasks/__init__.py b/pulp_deb/app/tasks/__init__.py index 2a1dd8e4f..e18e3af50 100644 --- a/pulp_deb/app/tasks/__init__.py +++ b/pulp_deb/app/tasks/__init__.py @@ -1,3 +1,4 @@ # flake8: noqa from .publishing import publish, publish_verbatim from .synchronizing import synchronize +from .copy import copy_content diff --git a/pulp_deb/app/tasks/copy.py b/pulp_deb/app/tasks/copy.py new file mode 100644 index 000000000..6f56bfc72 --- /dev/null +++ b/pulp_deb/app/tasks/copy.py @@ -0,0 +1,116 @@ +from django.db import transaction +from django.db.models import Q + +from pulpcore.plugin.models import Content, RepositoryVersion + +from pulp_deb.app.depsolving import Solver +from pulp_deb.app.models import AptRepository + +import logging +from gettext import gettext as _ + +log = logging.getLogger(__name__) + + +@transaction.atomic +def copy_content(config, dependency_solving=False): + """ + Copy content from one repo to another. + + Args: + source_repo_version_pk: repository version primary key to copy units from + dest_repo_pk: repository primary key to copy units into + criteria: a dict that maps type to a list of criteria to filter content by. Note that this + criteria MUST be validated before being passed to this task. + content_pks: a list of content pks to copy from source to destination + """ + + def process_entry(entry): + source_repo_version = RepositoryVersion.objects.get(pk=entry["source_repo_version"]) + dest_repo = AptRepository.objects.get(pk=entry["dest_repo"]) + + dest_version_provided = bool(entry.get("dest_base_version")) + if dest_version_provided: + dest_repo_version = RepositoryVersion.objects.get(pk=entry["dest_base_version"]) + else: + dest_repo_version = dest_repo.latest_version() + + if entry.get("content") is not None: + content_filter = Q(pk__in=entry.get("content")) + else: + content_filter = Q() + + log.info(_("Copying: {copy} created").format(copy=content_filter)) + + return ( + source_repo_version, + dest_repo_version, + dest_repo, + content_filter, + dest_version_provided, + ) + + if not dependency_solving: + # No Dependency Solving Branch + # ============================ + for entry in config: + ( + source_repo_version, + dest_repo_version, + dest_repo, + content_filter, + dest_version_provided, + ) = process_entry(entry) + + content_to_copy = source_repo_version.content.filter(content_filter) + + base_version = dest_repo_version if dest_version_provided else None + with dest_repo.new_version(base_version=base_version) as new_version: + new_version.add_content(content_to_copy) + else: + # Dependency Solving Branch + # ========================= + + # TODO: a more structured way to store this state would be nice. + content_to_copy = {} + repo_mapping = {} + libsolv_repo_names = {} + base_versions = {} + + solver = Solver() + + for entry in config: + ( + source_repo_version, + dest_repo_version, + dest_repo, + content_filter, + dest_version_provided, + ) = process_entry(entry) + + repo_mapping[source_repo_version] = dest_repo_version + base_versions[source_repo_version] = dest_version_provided + + # Load the content from the source and destination repository versions into the solver + source_repo_name = solver.load_source_repo(source_repo_version) + solver.load_target_repo(dest_repo_version) + + # Store the correspondance between the libsolv name of a repo version and the + # actual Pulp repo version, so that we can work backwards to get the latter + # from the former. + libsolv_repo_names[source_repo_name] = source_repo_version + + # Find all of the matching content in the repository version + content = source_repo_version.content.filter(content_filter) + content_to_copy[source_repo_name] = content + + solver.finalize() + + content_to_copy = solver.resolve_dependencies(content_to_copy) + + for from_repo, units in content_to_copy.items(): + src_repo_version = libsolv_repo_names[from_repo] + dest_repo_version = repo_mapping[src_repo_version] + base_version = dest_repo_version if base_versions[src_repo_version] else None + with dest_repo_version.repository.new_version(base_version=base_version) as new_version: + new_version.add_content(Content.objects.filter(pk__in=units)) diff --git a/pulp_deb/app/urls.py b/pulp_deb/app/urls.py new file mode 100644 index 000000000..5aed9ce68 --- /dev/null +++ b/pulp_deb/app/urls.py @@ -0,0 +1,5 @@ +from django.urls import path + +from .viewsets import CopyViewSet + +urlpatterns = [path("pulp/api/v3/deb/copy/", CopyViewSet.as_view({"post": "create"}))] diff --git a/pulp_deb/app/viewsets/__init__.py b/pulp_deb/app/viewsets/__init__.py index 8d87c66ab..b65a9a325 100644 --- a/pulp_deb/app/viewsets/__init__.py +++ b/pulp_deb/app/viewsets/__init__.py @@ -17,4 +17,4 @@ from .remote import AptRemoteViewSet -from .repository import AptRepositoryVersionViewSet, AptRepositoryViewSet +from .repository import AptRepositoryVersionViewSet, AptRepositoryViewSet, CopyViewSet diff --git a/pulp_deb/app/viewsets/repository.py b/pulp_deb/app/viewsets/repository.py index f2e885dc5..ce8aa6dc9 100644 --- a/pulp_deb/app/viewsets/repository.py +++ b/pulp_deb/app/viewsets/repository.py @@ -2,17 +2,21 @@ from drf_spectacular.utils import extend_schema from rest_framework.decorators import action +from rest_framework import viewsets +from rest_framework.serializers import ValidationError as DRFValidationError from pulpcore.plugin.actions import ModifyRepositoryActionMixin from pulpcore.plugin.serializers import ( AsyncOperationResponseSerializer, RepositorySyncURLSerializer, ) +from pulpcore.plugin.models import RepositoryVersion from pulpcore.plugin.tasking import dispatch from pulpcore.plugin.viewsets import ( OperationPostponedResponse, RepositoryVersionViewSet, RepositoryViewSet, + NamedModelViewSet, ) from pulp_deb.app import models, serializers, tasks @@ -57,11 +61,7 @@ def sync(self, request, pk): func=tasks.synchronize, exclusive_resources=[repository], shared_resources=[remote], - kwargs={ - "remote_pk": remote.pk, - "repository_pk": repository.pk, - "mirror": mirror, - }, + kwargs={"remote_pk": remote.pk, "repository_pk": repository.pk, "mirror": mirror}, ) return OperationPostponedResponse(result, request) @@ -76,3 +76,82 @@ class AptRepositoryVersionViewSet(RepositoryVersionViewSet): """ parent_viewset = AptRepositoryViewSet + + +class CopyViewSet(viewsets.ViewSet): + """ + ViewSet for Content Copy. + """ + + serializer_class = serializers.CopySerializer + + @extend_schema( + description="Trigger an asynchronous task to copy Apt content" + "from one repository into another, creating a new" + "repository version.", + summary="Copy content", + operation_id="copy_content", + request=serializers.CopySerializer, + responses={202: AsyncOperationResponseSerializer}, + ) + def create(self, request): + """Copy content.""" + serializer = serializers.CopySerializer(data=request.data, context={"request": request}) + serializer.is_valid(raise_exception=True) + + dependency_solving = serializer.validated_data["dependency_solving"] + config = serializer.validated_data["config"] + + config, shared_repos, exclusive_repos = self._process_config(config) + + async_result = dispatch( + tasks.copy_content, + shared_resources=shared_repos, + exclusive_resources=exclusive_repos, + args=[config, dependency_solving], + kwargs={}, + ) + return OperationPostponedResponse(async_result, request) + + def _process_config(self, config): + """ + Change the hrefs into pks within config. + This method also implicitly validates that the hrefs map to objects and it returns a list of + repos so that the task can lock on them. + """ + result = [] + # exclusive use of the destination repos is needed since new repository versions are being + # created, but source repos can be accessed in a read-only fashion in parallel, so long + # as there are no simultaneous modifications. + shared_repos = [] + exclusive_repos = [] + + for entry in config: + r = dict() + source_version = NamedModelViewSet().get_resource( + entry["source_repo_version"], RepositoryVersion + ) + dest_repo = NamedModelViewSet().get_resource(entry["dest_repo"], models.AptRepository) + r["source_repo_version"] = source_version.pk + r["dest_repo"] = dest_repo.pk + shared_repos.append(source_version.repository) + exclusive_repos.append(dest_repo) + + if "dest_base_version" in entry: + try: + r["dest_base_version"] = dest_repo.versions.get( + number=entry["dest_base_version"] + ).pk + except RepositoryVersion.DoesNotExist: + message = _( + "Version {version} does not exist for repository " "'{repo}'." + ).format(version=entry["dest_base_version"], repo=dest_repo.name) + raise DRFValidationError(detail=message) + + if entry.get("content") is not None: + r["content"] = [] + for c in entry["content"]: + r["content"].append(NamedModelViewSet().extract_pk(c)) + result.append(r) + + return result, shared_repos, exclusive_repos diff --git a/requirements.txt b/requirements.txt index ad43ec3c8..80e9aec3c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ pulpcore>=3.17.0.dev python-debian>=0.1.36 +solv~=0.7.17