Add a Nappy provider DAG using ProviderDataIngester

DAGs.md

@@ -67,6 +67,7 @@ The following are DAGs grouped by their primary tag:
 | [`jamendo_workflow`](#jamendo_workflow)                         | `@monthly`        | `False` | audio         |
 | [`metropolitan_museum_workflow`](#metropolitan_museum_workflow) | `@daily`          | `True`  | image         |
 | `museum_victoria_workflow`                                      | `@monthly`        | `False` | image         |
+| [`nappy_workflow`](#nappy_workflow)                             | `@monthly`        | `False` | image         |
 | `nypl_workflow`                                                 | `@monthly`        | `False` | image         |
 | [`phylopic_workflow`](#phylopic_workflow)                       | `@daily`          | `True`  | image         |
 | [`rawpixel_workflow`](#rawpixel_workflow)                       | `@monthly`        | `False` | image         |
@@ -105,6 +106,7 @@ The following is documentation associated with each DAG (where available):
 1.  [`jamendo_workflow`](#jamendo_workflow)
 1.  [`metropolitan_museum_reingestion_workflow`](#metropolitan_museum_reingestion_workflow)
 1.  [`metropolitan_museum_workflow`](#metropolitan_museum_workflow)
+1.  [`nappy_workflow`](#nappy_workflow)
 1.  [`oauth2_authorization`](#oauth2_authorization)
 1.  [`oauth2_token_refresh`](#oauth2_token_refresh)
 1.  [`phylopic_reingestion_workflow`](#phylopic_reingestion_workflow)
@@ -379,6 +381,17 @@ blocking during local development testing.
                         connect with just date and license.
                         https://collectionapi.metmuseum.org/public/collection/v1/search?isPublicDomain=true&metadataDate=2022-08-07
 
+## `nappy_workflow`
+
+Content Provider: Nappy
+
+ETL Process: Use the API to identify all CC0-licensed images.
+
+Output: TSV file containing the image meta-data.
+
+Notes: This api was written specially for Openverse. There are no known limits
+or restrictions. https://nappy.co/
+
 ## `oauth2_authorization`
 
 ### OAuth Provider Authorization

docker/local_postgres/0004_openledger_image_view.sql

@@ -12,9 +12,10 @@ INSERT INTO public.image_popularity_metrics (
   provider, metric, percentile
 ) VALUES
   ('flickr', 'views', 0.85),
-  ('wikimedia', 'global_usage_count', 0.85),
+  ('nappy', 'downloads', 0.85),
+  ('rawpixel', 'download_count', 0.85),
   ('stocksnap', 'downloads_raw', 0.85),
-  ('rawpixel', 'download_count', 0.85)
+  ('wikimedia', 'global_usage_count', 0.85)
 ;
 
 

openverse_catalog/dags/common/loader/provider_details.py

@@ -13,26 +13,27 @@
 
 
 # Default provider names
-FLICKR_DEFAULT_PROVIDER = "flickr"
-EUROPEANA_DEFAULT_PROVIDER = "europeana"
-WIKIMEDIA_AUDIO_PROVIDER = "wikimedia_audio"
-WIKIMEDIA_DEFAULT_PROVIDER = "wikimedia"
-SMITHSONIAN_DEFAULT_PROVIDER = "smithsonian"
 BROOKLYN_DEFAULT_PROVIDER = "brooklynmuseum"
 CLEVELAND_DEFAULT_PROVIDER = "clevelandmuseum"
+EUROPEANA_DEFAULT_PROVIDER = "europeana"
+FINNISH_DEFAULT_PROVIDER = "finnishmuseums"
+FLICKR_DEFAULT_PROVIDER = "flickr"
+FREESOUND_DEFAULT_PROVIDER = "freesound"
+INATURALIST_DEFAULT_PROVIDER = "inaturalist"
+JAMENDO_DEFAULT_PROVIDER = "jamendo"
 METROPOLITAN_MUSEUM_DEFAULT_PROVIDER = "met"
-VICTORIA_DEFAULT_PROVIDER = "museumsvictoria"
+NAPPY_DEFAULT_PROVIDER = "nappy"
 NYPL_DEFAULT_PROVIDER = "nypl"
 RAWPIXEL_DEFAULT_PROVIDER = "rawpixel"
 SCIENCE_DEFAULT_PROVIDER = "sciencemuseum"
+SMITHSONIAN_DEFAULT_PROVIDER = "smithsonian"
 SMK_DEFAULT_PROVIDER = "smk"
-WALTERS_DEFAULT_PROVIDER = "waltersartmuseum"
-FINNISH_DEFAULT_PROVIDER = "finnishmuseums"
-JAMENDO_DEFAULT_PROVIDER = "jamendo"
 STOCKSNAP_DEFAULT_PROVIDER = "stocksnap"
+VICTORIA_DEFAULT_PROVIDER = "museumsvictoria"
+WALTERS_DEFAULT_PROVIDER = "waltersartmuseum"
+WIKIMEDIA_AUDIO_PROVIDER = "wikimedia_audio"
+WIKIMEDIA_DEFAULT_PROVIDER = "wikimedia"
 WORDPRESS_DEFAULT_PROVIDER = "wordpress"
-FREESOUND_DEFAULT_PROVIDER = "freesound"
-INATURALIST_DEFAULT_PROVIDER = "inaturalist"
 PHYLOPIC_DEFAULT_PROVIDER = "phylopic"
 
 # Finnish parameters
@@ -138,6 +139,7 @@ class ImageCategory(Enum):
     "mccordmuseum": ImageCategory.DIGITIZED_ARTWORK.value,
     "met": ImageCategory.DIGITIZED_ARTWORK.value,
     "museumsvictoria": ImageCategory.DIGITIZED_ARTWORK.value,
+    "nappy": ImageCategory.PHOTOGRAPH.value,
     "phylopic": ImageCategory.ILLUSTRATION.value,
     "rijksmuseum": ImageCategory.DIGITIZED_ARTWORK.value,
     "sciencemuseum": ImageCategory.PHOTOGRAPH.value,

openverse_catalog/dags/common/popularity/sql.py

@@ -43,9 +43,10 @@
 
 IMAGE_POPULARITY_METRICS = {
     "flickr": {"metric": "views"},
-    "wikimedia": {"metric": "global_usage_count"},
-    "stocksnap": {"metric": "downloads_raw"},
+    "nappy": {"metric": "downloads"},
     "rawpixel": {"metric": "download_count"},
+    "stocksnap": {"metric": "downloads_raw"},
+    "wikimedia": {"metric": "global_usage_count"},
 }
 
 AUDIO_POPULARITY_METRICS = {

openverse_catalog/dags/providers/provider_api_scripts/nappy.py

@@ -0,0 +1,121 @@
+"""
+Content Provider:       Nappy
+
+ETL Process:            Use the API to identify all CC0-licensed images.
+
+Output:                 TSV file containing the image meta-data.
+
+Notes:                  This api was written specially for Openverse.
+                        There are no known limits or restrictions.
+                        https://nappy.co/
+
+"""
+import logging
+
+from common import constants
+from common.licenses import get_license_info
+from common.loader import provider_details as prov
+from providers.provider_api_scripts.provider_data_ingester import ProviderDataIngester
+
+
+logger = logging.getLogger(__name__)
+
+
+class NappyDataIngester(ProviderDataIngester):
+    providers = {constants.IMAGE: prov.NAPPY_DEFAULT_PROVIDER}
+    endpoint = "https://api.nappy.co/v1/openverse/images"
+    headers = {"User-Agent": prov.UA_STRING, "Accept": "application/json"}
+
+    # Hardoded to CC0, the only license Nappy.co uses
+    license_info = get_license_info(
+        "https://creativecommons.org/publicdomain/zero/1.0/"
+    )
+
+    def get_next_query_params(self, prev_query_params: dict | None, **kwargs) -> dict:
+        if not prev_query_params:
+            return {
+                "page": 1,
+                "per_page": self.batch_limit,
+            }
+        else:
+            return {
+                **prev_query_params,
+                "page": prev_query_params["page"] + 1,
+            }
+
+    def get_batch_data(self, response_json):
+        if response_json:
+            return response_json.get("images")
+        return None
+
+    def get_should_continue(self, response_json):
+        return bool(response_json.get("next_page"))
+
+    def get_media_type(self, record: dict):
+        return constants.IMAGE
+
+    @staticmethod
+    def _convert_filesize(raw_filesize_string: str) -> int:
+        """
+        Convert sizes from strings to byte integers, ex. "187.8kB" to 188.
+        """
+        FILETYPE_MULTIPLIERS = {"kB": 1000, "MB": 1_000_000, "GB": 1_000_000_000}
+        if isinstance(raw_filesize_string, str) and len(raw_filesize_string) > 2:
+            stripped = raw_filesize_string.strip()
+            if stripped[-2:] in FILETYPE_MULTIPLIERS:
+                try:
+                    units = float(stripped[:-2])
+                except ValueError:
+                    return
+                multiplier = FILETYPE_MULTIPLIERS[stripped[-2:]]
+                return round(units * multiplier)
+
+    def get_record_data(self, data: dict) -> dict | list[dict] | None:
+        if (foreign_landing_url := data.get("foreign_landing_url")) is None:
+            return None
+
+        if (image_url := data.get("url")) is None:
+            return None
+
+        foreign_identifier = data.get("foreign_identifier")
+        thumbnail_url = data.get("url") + "?auto=format&w=600&q=75"
+        filesize = self._convert_filesize(data.get("filesize"))
+        filetype = data.get("filetype")
+        creator = data.get("creator")
+        creator_url = data.get("creator_url")
+        title = data.get("title")
+        meta_data = {
+            "views": data.get("views"),
+            "saves": data.get("saves"),
+            "downloads": data.get("downloads"),
+        }
+        raw_tags = data.get("tags").split(",")
+        width = data.get("width")
+        height = data.get("height")
+
+        return {
+            "foreign_landing_url": foreign_landing_url,
+            "image_url": image_url,
+            "thumbnail_url": thumbnail_url,
+            "license_info": self.license_info,
+            "foreign_identifier": foreign_identifier,
+            "filesize": filesize,
+            "filetype": filetype,
+            "creator": creator,
+            "creator_url": creator_url,
+            "title": title,
+            "meta_data": meta_data,
+            "raw_tags": raw_tags,
+            "width": width,
+            "height": height,
+        }
+
+
+def main():
+    logger.info("Begin: Nappy data ingestion")
+    ingester = NappyDataIngester()
+    ingester.ingest_records()
+
+
+if __name__ == "__main__":
+    main()

openverse_catalog/dags/providers/provider_workflows.py

@@ -13,6 +13,7 @@
 from providers.provider_api_scripts.jamendo import JamendoDataIngester
 from providers.provider_api_scripts.metropolitan_museum import MetMuseumDataIngester
 from providers.provider_api_scripts.museum_victoria import VictoriaDataIngester
+from providers.provider_api_scripts.nappy import NappyDataIngester
 from providers.provider_api_scripts.nypl import NyplDataIngester
 from providers.provider_api_scripts.phylopic import PhylopicDataIngester
 from providers.provider_api_scripts.provider_data_ingester import ProviderDataIngester
@@ -160,6 +161,10 @@ def __post_init__(self):
         ingester_class=VictoriaDataIngester,
         start_date=datetime(2020, 1, 1),
     ),
+    ProviderWorkflow(
+        ingester_class=NappyDataIngester,
+        start_date=datetime(2022, 12, 1),
+    ),
     ProviderWorkflow(
         ingester_class=NyplDataIngester,
         start_date=datetime(2020, 1, 1),