Skip to content

Commit

Permalink
backend, frontend: implement project and build deletion in Pulp
Browse files Browse the repository at this point in the history 
Fix fedora-copr#3318
Fix fedora-copr#3319
  • Loading branch information
@FrostyX @nikromen
FrostyX authored and nikromen committed Sep 3, 2024
1 parent cfd0e8f commit 6d58166
Show file tree
Hide file tree
Showing 6 changed files with 229 additions and 90 deletions.
114 changes: 29 additions & 85 deletions backend/copr_backend/actions.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,7 @@
from .exceptions import CreateRepoError, CoprSignError, FrontendClientException
from .helpers import (get_redis_logger, silent_remove, ensure_dir_exists,
get_chroot_arch, format_filename,
uses_devel_repo, call_copr_repo, build_chroot_log_name,
copy2_but_hardlink_rpms)
call_copr_repo, copy2_but_hardlink_rpms)
from .sign import sign_rpms_in_dir, unsign_rpms_in_dir, get_pubkey


Expand Down Expand Up @@ -64,7 +63,6 @@ def get_action_class(cls, action):
ActionTypeEnum("rawhide_to_release"): RawhideToRelease,
ActionTypeEnum("fork"): Fork,
ActionTypeEnum("build_module"): BuildModule,
ActionTypeEnum("delete"): Delete,
ActionTypeEnum("remove_dirs"): RemoveDirs,
}.get(action_type, None)

Expand Down Expand Up @@ -221,72 +219,21 @@ def run(self):
return result


class Delete(Action):
"""
Abstract class for all other Delete* classes.
"""
# pylint: disable=abstract-method
def _handle_delete_builds(self, ownername, projectname, project_dirname,
chroot_builddirs, build_ids, appstream):
""" call /bin/copr-repo --delete """
devel = uses_devel_repo(self.front_url, ownername, projectname)
result = BackendResultEnum("success")
for chroot, subdirs in chroot_builddirs.items():
chroot_path = os.path.join(self.destdir, ownername, project_dirname,
chroot)
if not os.path.exists(chroot_path):
self.log.error("%s chroot path doesn't exist", chroot_path)
result = BackendResultEnum("failure")
continue

self.log.info("Deleting subdirs [%s] in %s",
", ".join(subdirs), chroot_path)

# Run createrepo first and then remove the files (to avoid old
# repodata temporarily pointing at non-existing files)!
if chroot != "srpm-builds":
# In srpm-builds we don't create repodata at all
if not call_copr_repo(chroot_path, delete=subdirs, devel=devel, appstream=appstream,
logger=self.log):
result = BackendResultEnum("failure")

for build_id in build_ids or []:
log_paths = [
os.path.join(chroot_path, build_chroot_log_name(build_id)),
# we used to create those before
os.path.join(chroot_path, 'build-{}.rsync.log'.format(build_id)),
os.path.join(chroot_path, 'build-{}.log'.format(build_id))]
for log_path in log_paths:
try:
os.unlink(log_path)
except OSError:
self.log.debug("can't remove %s", log_path)
return result


class DeleteProject(Delete):
class DeleteProject(Action):
def run(self):
self.log.debug("Action delete copr")
result = BackendResultEnum("success")

ext_data = json.loads(self.data["data"])
ownername = ext_data["ownername"]
project_dirnames = ext_data["project_dirnames"]
project_dirnames = self.ext_data["project_dirnames"]

if not ownername:
if not self.storage.owner:
self.log.error("Received empty ownername!")
result = BackendResultEnum("failure")
return result
return BackendResultEnum("failure")

for dirname in project_dirnames:
if not dirname:
self.log.warning("Received empty dirname!")
continue
path = os.path.join(self.destdir, ownername, dirname)
if os.path.exists(path):
self.log.info("Removing copr dir %s", path)
shutil.rmtree(path)
return result
self.storage.delete_project(dirname)
return BackendResultEnum("success")


class CompsUpdate(Action):
Expand Down Expand Up @@ -322,7 +269,7 @@ def run(self):
return result


class DeleteMultipleBuilds(Delete):
class DeleteMultipleBuilds(Action):
def run(self):
self.log.debug("Action delete multiple builds.")

Expand All @@ -334,25 +281,20 @@ def run(self):
# srpm-builds: [00849545, 00849546]
# fedora-30-x86_64: [00849545-example, 00849545-foo]
# [...]
ext_data = json.loads(self.data["data"])

ownername = ext_data["ownername"]
projectname = ext_data["projectname"]
project_dirnames = ext_data["project_dirnames"]
build_ids = ext_data["build_ids"]
appstream = ext_data["appstream"]
project_dirnames = self.ext_data["project_dirnames"]
build_ids = self.ext_data["build_ids"]

result = BackendResultEnum("success")
for project_dirname, chroot_builddirs in project_dirnames.items():
if BackendResultEnum("failure") == \
self._handle_delete_builds(ownername, projectname,
project_dirname, chroot_builddirs,
build_ids, appstream):
success = self.storage.delete_builds(
project_dirname, chroot_builddirs, build_ids)
if not success:
result = BackendResultEnum("failure")
return result


class DeleteBuild(Delete):
class DeleteBuild(Action):
def run(self):
self.log.info("Action delete build.")

Expand All @@ -363,25 +305,27 @@ def run(self):
# chroot_builddirs:
# srpm-builds: [00849545]
# fedora-30-x86_64: [00849545-example]
ext_data = json.loads(self.data["data"])

try:
ownername = ext_data["ownername"]
build_ids = [self.data['object_id']]
projectname = ext_data["projectname"]
project_dirname = ext_data["project_dirname"]
chroot_builddirs = ext_data["chroot_builddirs"]
appstream = ext_data["appstream"]
except KeyError:
valid = "object_id" in self.data
keys = {"ownername", "projectname", "project_dirname",
"chroot_builddirs", "appstream"}
for key in keys:
if key not in self.ext_data:
valid = False
break

if not valid:
self.log.exception("Invalid action data")
return BackendResultEnum("failure")

return self._handle_delete_builds(ownername, projectname,
project_dirname, chroot_builddirs,
build_ids, appstream)
success = self.storage.delete_builds(
self.ext_data["project_dirname"],
self.ext_data["chroot_builddirs"],
[self.data['object_id']])
return BackendResultEnum("success" if success else "failure")


class DeleteChroot(Delete):
class DeleteChroot(Action):
def run(self):
self.log.info("Action delete project chroot.")
chroot = self.ext_data["chrootname"]
Expand Down
61 changes: 61 additions & 0 deletions backend/copr_backend/pulp.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -157,6 +157,28 @@ def create_content(self, repository, path):
https://docs.pulpproject.org/pulp_rpm/restapi.html#tag/Content:-Packages/operation/content_rpm_packages_create
"""
url = self.url("api/v3/content/rpm/packages/")
data = {
"repository": repository,
"artifact": artifact,
"relative_path": relative_path,
}
return requests.post(url, json=data, **self.request_params)

def delete_content(self, repository, artifacts):
"""
Delete a list of artifacts from a repository
https://pulpproject.org/pulp_rpm/restapi/#tag/Repositories:-Rpm/operation/repositories_rpm_rpm_modify
"""
path = os.path.join(repository, "modify/")
url = self.config["base_url"] + path
data = {"remove_content_units": artifacts}
return requests.post(url, json=data, **self.request_params)

def upload_artifact(self, path):
"""
Create an artifact
https://docs.pulpproject.org/pulpcore/restapi.html#tag/Artifacts/operation/artifacts_create
"""
with open(path, "rb") as fp:
data = {"repository": repository}
files = {"file": fp}
Expand All @@ -178,3 +200,42 @@ def delete_distribution(self, distribution):
"""
url = self.config["base_url"] + distribution
return requests.delete(url, **self.request_params)

def list_packages(self, repository_version):
"""
List packages for a given repository version
Ideally, we would list packages for a repository but that probably
isn't possible.
TODO Since we don't care about repository versions, we should probably
set `--retain-repo-versions=1`
https://discourse.pulpproject.org/t/delete-artifact-with-pulp-cli/559/6
https://pulpproject.org/pulp_rpm/restapi/#tag/Content:-Packages/operation/content_rpm_packages_list
"""
url = self.url("api/v3/content/rpm/packages/?")
url += urlencode({"repository_version": repository_version})
return requests.get(url, **self.request_params)

def list_repositories(self, prefix):
"""
Get a list of repositories whose names match a given prefix
https://pulpproject.org/pulp_rpm/restapi/#tag/Repositories:-Rpm/operation/repositories_rpm_rpm_list
"""
url = self.url("api/v3/repositories/rpm/rpm/?")
url += urlencode({"name__startswith": prefix})
return requests.get(url, **self.request_params)

def get_latest_repository_version(self, name):
"""
Return the latest repository version
https://pulpproject.org/pulp_rpm/restapi/#tag/Repositories:-Rpm-Versions/operation/repositories_rpm_rpm_versions_list
"""
response = self.get_repository(name)
href = response.json()["results"][0]["versions_href"]

url = self.config["base_url"] + href + "?"
url += urlencode({"offset": 0, "limit": 1})

response = requests.get(url, **self.request_params)
return response.json()["results"][0]["pulp_href"]
Loading

0 comments on commit 6d58166

Please sign in to comment.