Refactor Rawpixel to use ProviderDataIngester

DAGs.md

@@ -86,7 +86,7 @@ The following are DAGs grouped by their primary tag:
 | `museum_victoria_workflow` | `@monthly` | `False` | image |
 | `nypl_workflow` | `@monthly` | `False` | image |
 | [`phylopic_workflow`](#phylopic_workflow) | `@daily` | `True` | image |
-| `rawpixel_workflow` | `@monthly` | `False` | image |
+| [`rawpixel_workflow`](#rawpixel_workflow) | `@monthly` | `False` | image |
 | `science_museum_workflow` | `@monthly` | `False` | image |
 | [`smithsonian_workflow`](#smithsonian_workflow) | `@weekly` | `False` | image |
 | `smk_workflow` | `@monthly` | `False` | image |
@@ -125,6 +125,7 @@ The following is documentation associated with each DAG (where available):
  1. [`oauth2_token_refresh`](#oauth2_token_refresh)
  1. [`phylopic_workflow`](#phylopic_workflow)
  1. [`pr_review_reminders`](#pr_review_reminders)
+ 1. [`rawpixel_workflow`](#rawpixel_workflow)
  1. [`recreate_audio_popularity_calculation`](#recreate_audio_popularity_calculation)
  1. [`recreate_image_popularity_calculation`](#recreate_image_popularity_calculation)
  1. [`report_pending_reported_media`](#report_pending_reported_media)
@@ -466,6 +467,23 @@ author of the PR to re-assign review if one of the randomly selected reviewers
 is unavailable for the time period during which the PR should be reviewed.
 
 
+## `rawpixel_workflow`
+
+
+Content Provider:       Rawpixel
+
+ETL Process:            Use the API to identify all CC-licensed images.
+
+Output:                 TSV file containing the image meta-data.
+
+Notes:                  Rawpixel has given us beta access to their API.
+                        This API is undocumented, and we will need to contact Rawpixel
+                        directly if we run into any issues.
+                        The public API max results range is limited to 100,000 results,
+                        although the API key we've been given can circumvent this limit.
+                        https://www.rawpixel.com/api/v1/search?tags=$publicdomain&page=1&pagesize=100
+
+
 ## `recreate_audio_popularity_calculation`
 
 

docker/local_postgres/0004_openledger_image_view.sql

@@ -5,12 +5,17 @@ CREATE TABLE public.image_popularity_metrics (
 );
 
 
+-- For more information on these values see:
+-- https://github.com/cc-archive/cccatalog/issues/405#issuecomment-629233047
+-- https://github.com/cc-archive/cccatalog/pull/477
 INSERT INTO public.image_popularity_metrics (
   provider, metric, percentile
 ) VALUES
   ('flickr', 'views', 0.85),
   ('wikimedia', 'global_usage_count', 0.85),
-  ('stocksnap', 'downloads_raw', 0.85);
+  ('stocksnap', 'downloads_raw', 0.85),
+  ('rawpixel', 'download_count', 0.85)
+;
 
 
 CREATE FUNCTION image_popularity_percentile(

env.template

@@ -28,10 +28,12 @@ AIRFLOW_VAR_API_KEY_BROOKLYN_MUSEUM=not_set
 AIRFLOW_VAR_API_KEY_DATA_GOV=not_set
 AIRFLOW_VAR_API_KEY_EUROPEANA=not_set
 AIRFLOW_VAR_API_KEY_FLICKR=not_set
+AIRFLOW_VAR_API_KEY_FREESOUND=not_set
 AIRFLOW_VAR_API_KEY_JAMENDO=not_set
 AIRFLOW_VAR_API_KEY_NYPL=not_set
+AIRFLOW_VAR_API_KEY_RAWPIXEL=not_set
 AIRFLOW_VAR_API_KEY_THINGIVERSE=not_set
-AIRFLOW_VAR_API_KEY_FREESOUND=not_set
+AIRFLOW_VAR_API_KEY_WALTERS_ART_MUSEUM=not_set
 
 
 ########################################################################################

openverse_catalog/dags/common/licenses/__init__.py

@@ -4,3 +4,6 @@
     get_license_info_from_license_pair,
     is_valid_license_info,
 )
+
+
+NO_LICENSE_FOUND = LicenseInfo(None, None, None, None)

openverse_catalog/dags/providers/provider_api_scripts/inaturalist.py

@@ -31,7 +31,7 @@
 from airflow.providers.postgres.operators.postgres import PostgresOperator
 from airflow.utils.task_group import TaskGroup
 from common.constants import POSTGRES_CONN_ID
-from common.licenses import LicenseInfo, get_license_info
+from common.licenses import NO_LICENSE_FOUND, get_license_info
 from common.loader import provider_details as prov
 from providers.provider_api_scripts.provider_data_ingester import ProviderDataIngester
 
@@ -87,7 +87,7 @@ def get_record_data(self, data):
             return None
         license_url = data.get("license_url")
         license_info = get_license_info(license_url=license_url)
-        if license_info == LicenseInfo(None, None, None, None):
+        if license_info == NO_LICENSE_FOUND:
             return None
         record_data = {k: data[k] for k in data.keys() if k != "license_url"}
         record_data["license_info"] = license_info