Unify pre-commit config across repos (#1006)

.git-blame-ignore-revs

@@ -0,0 +1 @@
+# Add pyupgrade

.pre-commit-config.yaml

@@ -1,10 +1,12 @@
-exclude: Pipfile\.lock|migrations|\.idea
+exclude: Pipfile\.lock|migrations|\.idea|node_modules|archive
 
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.0.1
     hooks:
       - id: trailing-whitespace
+        args: [--markdown-linebreak-ext=md]
+      - id: check-docstring-first
       - id: check-executables-have-shebangs
       - id: check-json
       - id: check-case-conflict
@@ -19,6 +21,7 @@ repos:
         args:
           - --remove
       - id: pretty-format-json
+        exclude: package(-lock)?\.json
         args:
           - --autofix
       - id: requirements-txt-fixer
@@ -29,6 +32,26 @@ repos:
       - id: isort
         files: \.py$
         exclude: ^build/.*$|^.tox/.*$|^venv/.*$
+        args:
+          - --lines-after-imports=2
+          - --multi-line=3
+          - --trailing-comma
+          - --force-grid-wrap=0
+          - --use-parentheses
+          - --ensure-newline-before-comments
+          - --line-length=88
+
+  - repo: https://github.com/asottile/pyupgrade
+    rev: v3.2.2
+    hooks:
+      - id: pyupgrade
+        args:
+          - --py310-plus
+
+  - repo: https://github.com/PyCQA/flake8
+    rev: 3.9.2
+    hooks:
+      - id: flake8
 
   - repo: https://github.com/ambv/black
     rev: 22.3.0
@@ -37,16 +60,23 @@ repos:
         args:
           - --safe
 
-  - repo: https://github.com/PyCQA/flake8
-    rev: 3.9.2
+  - repo: https://github.com/pre-commit/mirrors-eslint
+    rev: v8.3.0
     hooks:
-      - id: flake8
+      - id: eslint
+        files: ^js/.*$
+        additional_dependencies:
+          - [email protected]
+          - [email protected]
 
   - repo: https://github.com/pre-commit/mirrors-prettier
     rev: v2.5.0
     hooks:
+      - id: prettier
+        files: ^js/.*$
       - id: prettier
         types: [yaml]
+
   - repo: https://github.com/koalaman/shellcheck-precommit
     rev: v0.8.0
     hooks:

api/catalog/api/constants/field_order.py

@@ -45,4 +45,4 @@
 field_position_map: dict[str, int] = {
     field: idx for idx, field in enumerate(json_fields)
 }
-"""mapping of JSON fields to their sort positions"""
+#: mapping of JSON fields to their sort positions

api/catalog/api/controllers/search_controller.py

@@ -5,7 +5,7 @@
 import pprint
 from itertools import accumulate
 from math import ceil
-from typing import Any, List, Literal, Optional, Tuple
+from typing import Any, Literal
 
 from django.conf import settings
 from django.core.cache import cache
@@ -52,7 +52,7 @@ def _unmasked_query_end(page_size, page):
 
 def _paginate_with_dead_link_mask(
     s: Search, page_size: int, page: int
-) -> Tuple[int, int]:
+) -> tuple[int, int]:
     """
     Given a query, a page and page_size, return the start and end
     of the slice of results.
@@ -117,8 +117,8 @@ def _paginate_with_dead_link_mask(
 
 
 def _get_query_slice(
-    s: Search, page_size: int, page: int, filter_dead: Optional[bool] = False
-) -> Tuple[int, int]:
+    s: Search, page_size: int, page: int, filter_dead: bool | None = False
+) -> tuple[int, int]:
     """
     Select the start and end of the search results for this query.
     """
@@ -147,7 +147,7 @@ def _quote_escape(query_string):
 
 def _post_process_results(
     s, start, end, page_size, search_results, request, filter_dead
-) -> Optional[List[Hit]]:
+) -> list[Hit] | None:
     """
     After fetching the search results from the back end, iterate through the
     results, perform image validation, and route certain thumbnails through our
@@ -222,7 +222,7 @@ def _apply_filter(
     # Any is used here to avoid a circular import
     search_params: Any,  # MediaSearchRequestSerializer
     serializer_field: str,
-    es_field: Optional[str] = None,
+    es_field: str | None = None,
     behaviour: Literal["filter", "exclude"] = "filter",
 ):
     """
@@ -285,7 +285,7 @@ def search(
     request: Request,
     filter_dead: bool,
     page: int = 1,
-) -> Tuple[List[Hit], int, int]:
+) -> tuple[list[Hit], int, int]:
     """
     Given a set of keywords and an optional set of filters, perform a ranked
     paginated search.
@@ -489,8 +489,8 @@ def get_sources(index):
 
 
 def _get_result_and_page_count(
-    response_obj: Response, results: Optional[List[Hit]], page_size: int
-) -> Tuple[int, int]:
+    response_obj: Response, results: list[Hit] | None, page_size: int
+) -> tuple[int, int]:
     """
     Elasticsearch does not allow deep pagination of ranked queries.
     Adjust returned page count to reflect this.

api/catalog/api/models/audio.py

@@ -22,7 +22,7 @@ class AltAudioFile(AbstractAltFile):
     def __init__(self, attrs):
         self.bit_rate = attrs.get("bit_rate")
         self.sample_rate = attrs.get("sample_rate")
-        super(AltAudioFile, self).__init__(attrs)
+        super().__init__(attrs)
 
     @property
     def sample_rate_in_khz(self):
@@ -263,7 +263,7 @@ class Meta:
 
     @property
     def audio_url(self):
-        return super(AudioReport, self).url("audio")
+        return super().url("audio")
 
 
 class AudioList(AbstractMediaList):
@@ -278,4 +278,4 @@ class Meta:
 
     def save(self, *args, **kwargs):
         self.slug = uuslug(self.title, instance=self)
-        super(AudioList, self).save(*args, **kwargs)
+        super().save(*args, **kwargs)

api/catalog/api/models/image.py

@@ -84,7 +84,7 @@ class Meta:
 
     @property
     def image_url(self):
-        return super(ImageReport, self).url("photos")
+        return super().url("photos")
 
 
 class ImageList(AbstractMediaList):
@@ -99,4 +99,4 @@ class Meta:
 
     def save(self, *args, **kwargs):
         self.slug = uuslug(self.title, instance=self)
-        super(ImageList, self).save(*args, **kwargs)
+        super().save(*args, **kwargs)

api/catalog/api/models/media.py

@@ -216,7 +216,7 @@ def save(self, *args, **kwargs):
             same_reports = same_reports.filter(reason=self.reason)
         same_reports.update(status=self.status)
 
-        super(AbstractMediaReport, self).save(*args, **kwargs)
+        super().save(*args, **kwargs)
 
 
 class AbstractDeletedMedia(OpenLedgerModel):

api/catalog/api/utils/attribution.py

@@ -4,17 +4,15 @@
 frontend, or open an issue to track it.
 """
 
-from typing import Optional
-
 from catalog.api.utils.licenses import get_full_license_name, is_public_domain
 
 
 def get_attribution_text(
-    title: Optional[str],
-    creator: Optional[str],
+    title: str | None,
+    creator: str | None,
     _license: str,
-    license_version: Optional[str],
-    license_url: Optional[str],
+    license_version: str | None,
+    license_url: str | None,
 ) -> str:
     """
     Get the attribution text to properly and legally attribute a creative work to its

api/catalog/api/utils/ccrel.py

@@ -1,11 +1,3 @@
-import io
-import os
-import uuid
-
-import libxmp
-from libxmp.consts import XMP_NS_CC, XMP_NS_XMP, XMP_NS_XMP_Rights
-
-
 """
 Tools for embedding Creative Commons Rights Expression Language (ccREL) data
 into files using Extensible Metadata Platform (XMP).
@@ -17,6 +9,13 @@
 [0] https://www.w3.org/Submission/ccREL/
 """
 
+import io
+import os
+import uuid
+
+import libxmp
+from libxmp.consts import XMP_NS_CC, XMP_NS_XMP, XMP_NS_XMP_Rights
+
 
 def embed_xmp_bytes(image: io.BytesIO, work_properties):
     """

api/catalog/api/utils/dead_link_mask.py

@@ -1,5 +1,3 @@
-from typing import List
-
 from deepdiff import DeepHash
 from django_redis import get_redis_connection
 from elasticsearch_dsl import Search
@@ -25,7 +23,7 @@ def get_query_hash(s: Search) -> str:
     return deep_hash
 
 
-def get_query_mask(query_hash: str) -> List[int]:
+def get_query_mask(query_hash: str) -> list[int]:
     """
     Fetches an existing query mask for a given query hash
     or returns an empty one.
@@ -38,7 +36,7 @@ def get_query_mask(query_hash: str) -> List[int]:
     return list(map(int, redis.lrange(key, 0, -1)))
 
 
-def save_query_mask(query_hash: str, mask: List):
+def save_query_mask(query_hash: str, mask: list):
     """
     Saves a query mask to redis.
 

api/catalog/api/utils/help_text.py

@@ -1,4 +1,4 @@
-from typing import Iterable
+from collections.abc import Iterable
 
 
 def make_comma_separated_help_text(items: Iterable[str], name: str) -> str:

api/catalog/api/utils/licenses.py

@@ -4,7 +4,6 @@
 frontend, or open an issue to track it.
 """
 
-from typing import Optional
 
 from catalog.api.constants.licenses import (
     ALL_CC_LICENSES,
@@ -13,7 +12,7 @@
 )
 
 
-def get_license_url(_license: str, license_version: Optional[str]) -> str:
+def get_license_url(_license: str, license_version: str | None) -> str:
     """
     Get the URL to the deed of the license.
 
@@ -33,7 +32,7 @@ def get_license_url(_license: str, license_version: Optional[str]) -> str:
     return f"https://creativecommons.org/{fragment}/"
 
 
-def get_full_license_name(_license: str, license_version: Optional[str]) -> str:
+def get_full_license_name(_license: str, license_version: str | None) -> str:
     """
     Get the full name of the license in a displayable format from the license
     slug and version.

api/catalog/api/utils/waveform.py

@@ -6,7 +6,6 @@
 import pathlib
 import shutil
 import subprocess
-from typing import List
 
 from django.conf import settings
 
@@ -141,7 +140,7 @@ def cleanup(file_name):
         logger.debug("file not found, nothing deleted")
 
 
-def generate_peaks(audio) -> List[float]:
+def generate_peaks(audio) -> list[float]:
     file_name = None
     try:
         file_name = download_audio(audio.url, audio.identifier)

api/catalog/configuration/elasticsearch.py

@@ -1,3 +1,7 @@
+"""
+This file contains configuration pertaining to Elasticsearch.
+"""
+
 from django.conf import settings
 
 from aws_requests_auth.aws_auth import AWSRequestsAuth
@@ -42,11 +46,12 @@ def _elasticsearch_connect():
 
 
 ES = _elasticsearch_connect()
-"""Elasticsearch client, also aliased to connection 'default'"""
+#: Elasticsearch client, also aliased to connection 'default'
+
 connections.add_connection("default", ES)
 
 MEDIA_INDEX_MAPPING = {
     media_type: config(f"{media_type.upper()}_INDEX_NAME", default=media_type)
     for media_type in MEDIA_TYPES
 }
-"""mapping of media types to Elasticsearch index names"""
+#: mapping of media types to Elasticsearch index names

api/catalog/configuration/link_validation_cache.py

@@ -3,7 +3,6 @@
 import os
 from collections import defaultdict
 from datetime import timedelta
-from typing import Optional
 
 from django.core.exceptions import ImproperlyConfigured
 
@@ -53,7 +52,7 @@ def __init__(self):
 
             self[status] = value
 
-    def _config(self, key: str | int, default: Optional[dict] = None) -> Optional[int]:
+    def _config(self, key: str | int, default: dict | None = None) -> int | None:
         try:
             v = config(
                 f"{self.SETTING_PREFIX}{str(key)}",

api/catalog/urls/swagger.py

@@ -12,7 +12,7 @@
     "docs",
     "README.md",
 )
-with open(description_path, "r") as description_file:
+with open(description_path) as description_file:
     description = description_file.read()
 
 tos_url = "https://wordpress.github.io/openverse-api/terms_of_service.html"