diff --git a/.pylintrc b/.pylintrc new file mode 100644 index 0000000..8e74d1d --- /dev/null +++ b/.pylintrc @@ -0,0 +1,52 @@ +[MASTER] +load-plugins=pylint_quotes + +string-quote=double-avoid-escape +triple-quote=double +docstring-quote=double + +[MESSAGES CONTROL] +disable= + abstract-method, + bad-continuation, + chained-comparison, # R1716: Simplify chained comparison between the operands + duplicate-code, + fixme, + invalid-name, + len-as-condition, + missing-docstring, + no-else-return, + no-else-raise, + no-self-use, + superfluous-parens, + too-few-public-methods, + too-many-ancestors, + too-many-arguments, + too-many-boolean-expressions, + too-many-branches, + too-many-instance-attributes, + too-many-lines, + too-many-locals, + too-many-nested-blocks, + too-many-public-methods, + too-many-statements, + ungrouped-imports, + unused-argument, + wrong-import-order, + line-too-long, + no-else-continue, + no-else-break, + import-outside-toplevel + +[FORMAT] +max-line-length=125 + +[REPORTS] +output-format=text +reports=no +score=no + +[TYPECHECK] +ignored-classes=PurePath,responses +extension-pkg-whitelist=cassandra.cluster,cassandra.metadata,cassandra.query + diff --git a/Makefile b/Makefile index feb09da..a775e4d 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ short_ver = 0.0.1 long_ver = $(shell git describe --long 2>/dev/null || echo $(short_ver)-0-unknown-g`git describe --always`) -all: +all: PYTHON ?= python3 PYTHON_SOURCE_DIRS = rpm_s3_mirror/ tests/ @@ -28,7 +28,7 @@ build-dep-fed: python3-boto3 \ python3-lxml -test: copyright unittest +test: copyright pylint unittest .PHONY: copyright copyright: diff --git a/rpm_s3_mirror/__main__.py b/rpm_s3_mirror/__main__.py index 8388787..40a60a6 100644 --- a/rpm_s3_mirror/__main__.py +++ b/rpm_s3_mirror/__main__.py @@ -6,9 +6,9 @@ from rpm_s3_mirror.config import JSONConfig, ENVConfig from rpm_s3_mirror.mirror import Mirror -logging.getLogger('boto').setLevel(logging.WARNING) -logging.getLogger('botocore').setLevel(logging.WARNING) -logging.getLogger('urllib3').setLevel(logging.WARNING) +logging.getLogger("boto").setLevel(logging.WARNING) +logging.getLogger("botocore").setLevel(logging.WARNING) +logging.getLogger("urllib3").setLevel(logging.WARNING) logging.basicConfig(level=logging.DEBUG) diff --git a/rpm_s3_mirror/config.py b/rpm_s3_mirror/config.py index 3b2db38..f14b57e 100644 --- a/rpm_s3_mirror/config.py +++ b/rpm_s3_mirror/config.py @@ -70,7 +70,7 @@ def _populate_required(self): elif key == "max_workers": value = int(value) elif key == "bootstrap": - value = True if value.lower() == "true" else False + value = value.lower() == "true" self._config[key] = value diff --git a/rpm_s3_mirror/mirror.py b/rpm_s3_mirror/mirror.py index c87784d..c41e6c8 100644 --- a/rpm_s3_mirror/mirror.py +++ b/rpm_s3_mirror/mirror.py @@ -48,7 +48,7 @@ def sync(self): mirror_repository = RPMRepository(base_url=self._build_s3_url(upstream_repository)) last_check_time = self.s3.repomd_update_time(base_url=mirror_repository.base_url) if not upstream_repository.has_updates(since=last_check_time): - self.log.info(f"Skipping repository with no updates since: {last_check_time}") + self.log.info("Skipping repository with no updates since: %s", last_check_time) continue # Extract our metadata and detect any new/updated packages. diff --git a/rpm_s3_mirror/repository.py b/rpm_s3_mirror/repository.py index c8c67d2..3af1c67 100644 --- a/rpm_s3_mirror/repository.py +++ b/rpm_s3_mirror/repository.py @@ -5,8 +5,8 @@ from typing import Iterator, Dict from urllib.parse import urlparse -from lxml.etree import fromstring, Element -from lxml.etree import XMLParser +from lxml.etree import fromstring, Element # pylint: disable=no-name-in-module +from lxml.etree import XMLParser # pylint: disable=no-name-in-module from dateutil.parser import parse from tempfile import TemporaryDirectory import os @@ -33,7 +33,7 @@ def safe_parse_xml(xml_string: bytes) -> Element: def download_repodata_section(section, request, destination_dir) -> str: local_path = join(destination_dir, os.path.basename(section.location)) - with open(local_path, 'wb') as out: + with open(local_path, "wb") as out: shutil.copyfileobj(request.raw, out) validate_checksum(path=local_path, checksum_type=section.checksum_type, checksum=section.checksum) return local_path @@ -101,7 +101,7 @@ def __init__(self, base_url: str, packages_xml: bytes): self.root = safe_parse_xml(packages_xml) def __len__(self) -> int: - return int(self.root.get('packages')) + return int(self.root.get("packages")) def __iter__(self) -> Iterator[Package]: for package_element in self.root: @@ -145,12 +145,12 @@ def _extract_package_list(self, primary: RepodataSection) -> PackageList: def parse_repomd(self, xml: Element) -> Dict[str, RepodataSection]: sections = {} - for data_element in xml.findall(f'repo:data', namespaces=namespaces): + for data_element in xml.findall(f"repo:data", namespaces=namespaces): section_type = data_element.attrib["type"] section = {} - for element in xml.findall(f'repo:data[@type="{section_type}"]/repo:*', namespaces=namespaces): + for element in xml.findall(f"repo:data[@type='{section_type}']/repo:*", namespaces=namespaces): # Strip the namespace from the tag as it is annoying - _, _, key = element.tag.partition('}') + _, _, key = element.tag.partition("}") value = element.text if key == "location": value = element.get("href") diff --git a/rpm_s3_mirror/s3.py b/rpm_s3_mirror/s3.py index 038a854..4c33379 100644 --- a/rpm_s3_mirror/s3.py +++ b/rpm_s3_mirror/s3.py @@ -113,13 +113,13 @@ def repomd_update_time(self, base_url: str) -> datetime: def _sync_objects(self, temp_dir: str, repo_objects: Iterable[Package], skip_existing: bool): sync = functools.partial(self._sync_object, temp_dir, skip_existing) start = time.time() - self.log.info(f"Beginning sync of {len(repo_objects)} objects.") + self.log.info("Beginning sync of %s objects.", len(repo_objects)) with ThreadPoolExecutor(max_workers=self.max_workers) as executor: # We iterate through the generator to pick up and propagate any Exceptions for _ in executor.map(sync, repo_objects): pass elapsed = int(time.time() - start) - self.log.info(f"Completed syncing {len(repo_objects)} objects in {elapsed} seconds") + self.log.info("Completed syncing %s objects in %s seconds", len(repo_objects), elapsed) def _sync_object(self, temp_dir: str, skip_existing: bool, repo_object: Union[Package, RepodataSection]): if skip_existing and self._object_exists(repo_object.destination): @@ -131,7 +131,7 @@ def _sync_object(self, temp_dir: str, skip_existing: bool, repo_object: Union[Pa self._put_object(package_path, repo_object.destination) try: os.unlink(package_path) - except Exception as e: + except Exception as e: # pylint: disable=broad-except self.log.debug("Failed to unlink %s: %s", package_path, e) def _download_file(self, temp_dir: str, url: str) -> str: @@ -139,7 +139,7 @@ def _download_file(self, temp_dir: str, url: str) -> str: with self.session.get(url, stream=True) as request: request.raise_for_status() out_path = join(temp_dir, os.path.basename(url)) - with open(out_path, 'wb') as f: + with open(out_path, "wb") as f: shutil.copyfileobj(request.raw, f) return out_path @@ -224,4 +224,4 @@ def _build_md5_header(self, fp: BinaryIO) -> str: while data: h.update(data) data = fp.read(1000000) - return base64.b64encode(h.digest()).decode('utf-8') + return base64.b64encode(h.digest()).decode("utf-8") diff --git a/rpm_s3_mirror/statsd.py b/rpm_s3_mirror/statsd.py index 9138196..ff9ad17 100644 --- a/rpm_s3_mirror/statsd.py +++ b/rpm_s3_mirror/statsd.py @@ -18,12 +18,12 @@ def __init__( host="127.0.0.1", port=8125, tags=None, - format="telegraf", + metric_format="telegraf", ): self._dest_addr = (host, port) self._socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) self._tags = tags or {} - self._message_format = format + self._message_format = metric_format def gauge(self, metric, value, tags=None): self._send(metric, b"g", value, tags) diff --git a/rpm_s3_mirror/util.py b/rpm_s3_mirror/util.py index a29dee1..c21f862 100644 --- a/rpm_s3_mirror/util.py +++ b/rpm_s3_mirror/util.py @@ -20,8 +20,8 @@ def validate_checksum(path, checksum_type, checksum) -> None: def get_requests_session() -> Session: session = requests.session() retries = Retry(total=5, backoff_factor=0.1, status_forcelist=[500, 502, 503, 504]) - session.mount('http://', HTTPAdapter(max_retries=retries)) - session.mount('https://', HTTPAdapter(max_retries=retries)) + session.mount("http://", HTTPAdapter(max_retries=retries)) + session.mount("https://", HTTPAdapter(max_retries=retries)) return session diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..dc47a8d --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1 @@ +# Copyright (c) 2020 Aiven, Helsinki, Finland. https://aiven.io/ diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..38d34be --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,12 @@ +# Copyright (c) 2020 Aiven, Helsinki, Finland. https://aiven.io/ + +import os +import pytest + + +@pytest.fixture(name="test_package_list_xml") +def test_package_list_xml(): + root_dir = os.path.dirname(os.path.abspath(__file__)) + package_xml_path = os.path.join(root_dir, "resources", "primary.xml") + with open(package_xml_path, "rb") as f: + return f.read() diff --git a/tests/resources/primary.xml b/tests/resources/primary.xml new file mode 100644 index 0000000..f7ddd30 --- /dev/null +++ b/tests/resources/primary.xml @@ -0,0 +1,931 @@ + + + GMT + i686 + + e0445c5a0b1241058376f0dac30baceb07fbd2765e2fc0706d87121f788bf4b8 + Generic Mapping Tools + GMT is an open source collection of ~60 tools for manipulating geographic and +Cartesian data sets (including filtering, trend fitting, gridding, projecting, +etc.) and producing Encapsulated PostScript File (EPS) illustrations ranging +from simple x-y plots via contour maps to artificially illuminated surfaces +and 3-D perspective views. GMT supports ~30 map projections and transforma- +tions and comes with support data such as coastlines, rivers, and political +boundaries. + +GMT is developed and maintained by Paul Wessel and Walter H. F. Smith with +help from a global set of volunteers, and is supported by the National +Science Foundation. + +NOTE: Specific executables that conflict with other Fedora packages have been +removed. These functions can still be accessed via the GMT wrapper script +with: GMT <function> [args] + Fedora Project + https://www.generic-mapping-tools.org/ + + + GMT-doc + noarch + + 4c0139b19b457bff1f0307ad682fa8908e476569115ed8cced5ece7d27aeca1b + Documentation for GMT + The GMT-doc package provides the documentation for the GMT (Generic +Mapping Tools) package. + Fedora Project + https://www.generic-mapping-tools.org/ + + + GraphicsMagick-c++ + i686 + + 784424a83f98769749aede05e2f7bda122cb5600b3443cfceb9f1d83b17ae12c + GraphicsMagick Magick++ library (C++ bindings) + This package contains the GraphicsMagick++ library, a C++ binding to the +GraphicsMagick graphics manipulation library. + +Install GraphicsMagick-c++ if you want to use any applications that use +GraphicsMagick++. + Fedora Project + http://www.graphicsmagick.org/ + + + GraphicsMagick-devel + i686 + + f8d8829a4e98d19dedf755f0ff0b5b499dcd1bf0d52056da54eb5b77638ba3c4 + Libraries and header files for GraphicsMagick app development + GraphicsMagick-devel contains the Libraries and header files you'll +need to develop GraphicsMagick applications. GraphicsMagick is an image +manipulation program. + +If you want to create applications that will use GraphicsMagick code or +APIs, you need to install GraphicsMagick-devel as well as GraphicsMagick. +You do not need to install it if you just want to use GraphicsMagick, +however. + Fedora Project + http://www.graphicsmagick.org/ + + + ImageMagick + i686 + + 880e700da36911f0c008ed73082fdedf01ee1b771688e883f3565921cd2f4b6f + An X application for displaying and manipulating images + ImageMagick is an image display and manipulation tool for the X +Window System. ImageMagick can read and write JPEG, TIFF, PNM, GIF, +and Photo CD image formats. It can resize, rotate, sharpen, color +reduce, or add special effects to an image, and when finished you can +either save the completed work in the original format or a different +one. ImageMagick also includes command line programs for creating +animated or transparent .gifs, creating composite images, creating +thumbnail images, and more. + +ImageMagick is one of your choices if you need a program to manipulate +and display images. If you want to develop your own applications +which use ImageMagick code or APIs, you need to install +ImageMagick-devel as well. + Fedora Project + http://www.imagemagick.org/ + + + ImageMagick-c++-devel + i686 + + 9c22525ce337817559595aae5a6d5b9d91ae3c6405fa26ec42e875e43d2b7063 + C++ bindings for the ImageMagick library + ImageMagick-devel contains the static libraries and header files you'll +need to develop ImageMagick applications using the Magick++ C++ bindings. +ImageMagick is an image manipulation program. + +If you want to create applications that will use Magick++ code +or APIs, you'll need to install ImageMagick-c++-devel, ImageMagick-devel and +ImageMagick. +You don't need to install it if you just want to use ImageMagick, or if you +want to develop/compile applications using the ImageMagick C interface, +however. + Fedora Project + http://www.imagemagick.org/ + + + ImageMagick-devel + i686 + + e802d9d12bb6cc8ffe06589c20b9fb44f458fb26b6119aeeb2fc8c13a5f98658 + Library links and header files for ImageMagick app development + ImageMagick-devel contains the library links and header files you'll +need to develop ImageMagick applications. ImageMagick is an image +manipulation program. + +If you want to create applications that will use ImageMagick code or +APIs, you need to install ImageMagick-devel as well as ImageMagick. +You do not need to install it if you just want to use ImageMagick, +however. + Fedora Project + http://www.imagemagick.org/ + + + InsightToolkit-vtk-devel + i686 + + 307153dada4114e189f73dc40882eae16f7bb543631948f7697348aebfd038be + Libraries and header files for development of ITK-VTK bridge + Libraries and header files for development of ITK-VTK bridge + Fedora Project + https://www.itk.org/ + + + PackageKit-cron + x86_64 + + c9d736ccabfe4fe3d2a7daf2c5430bf146732a701e74e32c0882e531b7a0c65e + Cron job and related utilities for PackageKit + Crontab and utilities for running PackageKit as a cron job. + Fedora Project + http://www.freedesktop.org/software/PackageKit/ + + + R-R.devices + noarch + + aba6e8ab8b05f49c7b8ecf9fd94c89c73be88e90a5f1bf91bd3107a816444bb6 + Unified Handling of Graphics Devices + Functions for creating plots and image files in a unified way regardless of +output format (EPS, PDF, PNG, SVG, TIFF, WMF, etc.). Default device options as +well as scales and aspect ratios are controlled in a uniform way across all +device types. Switching output format requires minimal changes in code. This +package is ideal for large-scale batch processing, because it will never leave +open graphics devices or incomplete image files behind, even on errors or user +interrupts. + Fedora Project + https://CRAN.R-project.org/package=R.devices + + + R-R.rsp + noarch + + 5479db59076be1f7c660059a011d5afce42dc2e351838b3346ae8d1ce3250d78 + Dynamic Generation of Scientific Reports + The RSP markup language makes any text-based document come alive. RSP provides +a powerful markup for controlling the content and output of LaTeX, HTML, +Markdown, AsciiDoc, Sweave and knitr documents (and more), e.g. 'Today's date +is <%=Sys.Date()%>'. Contrary to many other literate programming languages, +with RSP it is straightforward to loop over mixtures of code and text sections, +e.g. in month-by-month summaries. RSP has also several preprocessing +directives for incorporating static and dynamic contents of external files +(local or online) among other things. Functions rstring() and rcat() make it +easy to process RSP strings, rsource() sources an RSP file as it was an R +script, while rfile() compiles it (even online) into its final output format, +e.g. rfile('report.tex.rsp') generates 'report.pdf' and rfile('report.md.rsp') +generates 'report.html'. RSP is ideal for self-contained scientific reports +and R package vignettes. It's easy to use - if you know how to write an R +script, you'll be up and running within minutes. + Fedora Project + https://CRAN.R-project.org/package=R.rsp + + + R-RInside + x86_64 + + 761c0f02a04b17695f1b24d8f19f816803cf8729eb811c7899504a0f0f9e5b1a + C++ Classes to Embed R in C++ (and C) Applications + The RInside packages makes it easier to have "R inside" your C++ +application by providing a C++ wrapper class providing the R +interpreter. + Fedora Project + https://cran.r-project.org/package=RInside + + + R-RInside-examples + x86_64 + + 159e3bfeb065bc5875eaf7b034bcfea0f3cc0228e61afa616c3a695b4a003453 + RInside Examples + Numerous examples are provided in the nine sub-directories of the +examples directory of the installed package: standard, mpi (for +parallel computing), qt (showing how to embed RInside inside a Qt GUI +application), wt (showing how to build a "web-application" using the +Wt toolkit), armadillo (for RInside use with RcppArmadillo), eigen +(for RInside use with RcppEigen) and 'c_interface' for a basic C +interface and 'Ruby' illustration. + Fedora Project + https://cran.r-project.org/package=RInside + + + R-XVector-devel + i686 + + 77ad6cd60e026dbf3cf5c106091bbf7016f791bcb8d4c4e5718b0bb5b94673fe + Development files for R-XVector + Development files for R-XVector. + Fedora Project + http://www.bioconductor.org/packages/release/bioc/html/XVector.html + + + R-backports + x86_64 + + 09dae965475d8be80a958ddee560eecf2e0c98333443eccc84a66938afb56667 + Reimplementations of Functions Introduced Since R-3.0.0 + Functions introduced or changed since R v3.0.0 are re-implemented in this +package. The backports are conditionally exported in order to let R resolve +the function name to either the implemented backport, or the respective +base version, if available. Package developers can make use of new +functions or arguments by selectively importing specific backports to +support older installations. + Fedora Project + https://CRAN.R-project.org/package=backports + + + R-callr + noarch + + be758223d0078ea2da7bfb7fc0fac3bf465da52b657a4e9fd3129d04d241a5f6 + Call R from R + It is sometimes useful to perform a computation in a separate R process, +without affecting the current R process at all. This packages does exactly +that. + Fedora Project + https://CRAN.R-project.org/package=callr + + + R-cli + noarch + + d3d3a4751f51a4eb6aba37dd1631f9471418ec4dc7f247cf05e09150ce1359ae + Helpers for Developing Command Line Interfaces + A suite of tools to build attractive command line interfaces ('CLIs'), from +semantic elements: headings, lists, alerts, paragraphs, etc. Supports custom +themes via a 'CSS'-like language. It also contains a number of lower level +'CLI' elements: rules, boxes, trees, and 'Unicode' symbols with 'ASCII' +alternatives. It integrates with the 'crayon' package to support 'ANSI' +terminal colors. + Fedora Project + https://CRAN.R-project.org/package=cli + + + R-curl + x86_64 + + 826f4de1836c0c9c1664a773fce4d694bd6adaeaf9edd7ff684468187333125c + A Modern and Flexible Web Client for R + The curl() and curl_download() functions provide highly configurable drop-in +replacements for base url() and download.file() with better performance, +support for encryption (https, ftps), gzip compression, authentication, and +other 'libcurl' goodies. The core of the package implements a framework for +performing fully customized requests where data can be processed either in +memory, on disk, or streaming via the callback or connection interfaces. Some +knowledge of 'libcurl' is recommended; for a more-user-friendly web client see +the 'httr' package which builds on this package with http specific tools and +logic. + Fedora Project + https://CRAN.R-project.org/package=curl + + + R-cyclocomp + noarch + + eb4574d3fe68d8bff8ee663bc6ae7d4d688fe8e63d9b7546bbeace948458bae5 + Cyclomatic Complexity of R Code + Cyclomatic complexity is a software metric (measurement), used to indicate +the complexity of a program. It is a quantitative measure of the number of +linearly independent paths through a program's source code. It was +developed by Thomas J. McCabe, Sr. in 1976. + Fedora Project + https://CRAN.R-project.org/package=cyclocomp + + + R-deldir + x86_64 + + 6d9c130810333486a3b6916bb483a1e2b117779aa327b0a84a9891199dad43cf + Delaunay Triangulation and Dirichlet (Voronoi) Tessellation + Calculates the Delaunay triangulation and the Dirichlet or Voronoi +tessellation (with respect to the entire plane) of a planar point set. +Plots triangulations and tessellations in various ways. Clips +tessellations to sub-windows. Calculates perimeters of tessellations. +Summarises information about the tiles of the tessellation. + Fedora Project + https://CRAN.R-project.org/package=deldir + + diff --git a/tests/test_package.py b/tests/test_package.py new file mode 100644 index 0000000..9556bb2 --- /dev/null +++ b/tests/test_package.py @@ -0,0 +1,11 @@ +# Copyright (c) 2020 Aiven, Helsinki, Finland. https://aiven.io/ + +from rpm_s3_mirror.repository import Package, PackageList + + +def test_package_list(test_package_list_xml): + package_list = PackageList(base_url="https://some.repo/some/path", packages_xml=test_package_list_xml) + packages = list(package_list) + assert len(package_list) > 0 + assert len(package_list) == len(packages) + assert all((isinstance(package, Package) for package in packages))