From aef5a1b9f5d9841159e5610c828fa4bbbd1b33ff Mon Sep 17 00:00:00 2001 From: sarayourfriend Date: Wed, 26 Jun 2024 11:39:03 +1000 Subject: [PATCH] Add DAG to decode and deduplicate image tags with escaped literal unicode sequences (#4475) * Add (unusable) DAG to decode and deduplicate tags * Only modify tags with escaped unicode sequences * Fix timeouts and pass param validation --- .../decode_and_deduplicate_image_tags.py | 127 ++++++++++++++++++ .../maintenance/trim_and_deduplicate_tags.py | 4 +- documentation/catalog/reference/DAGs.md | 17 +++ sample_data/sample_image.csv | 2 +- 4 files changed, 147 insertions(+), 3 deletions(-) create mode 100644 catalog/dags/maintenance/decode_and_deduplicate_image_tags.py diff --git a/catalog/dags/maintenance/decode_and_deduplicate_image_tags.py b/catalog/dags/maintenance/decode_and_deduplicate_image_tags.py new file mode 100644 index 00000000000..c5e728283e7 --- /dev/null +++ b/catalog/dags/maintenance/decode_and_deduplicate_image_tags.py @@ -0,0 +1,127 @@ +""" +See the issue for context and motivation: https://github.com/WordPress/openverse/issues/4452 + +This DAG triggers a run of the batched update DAG. It generates a new list of tags by +trimming all existing tags and re-inserting only the distinct tags of the resulting list of tags. + +Only records before the CC Search -> Openverse transition are affected. As such, because all +audio records are dated after that transition, we only need to scan images. +""" + +from datetime import datetime, timedelta +from textwrap import dedent + +from airflow.decorators import dag, task +from airflow.models.abstractoperator import AbstractOperator +from airflow.operators.trigger_dagrun import TriggerDagRunOperator + +from common.constants import DAG_DEFAULT_ARGS, POSTGRES_CONN_ID +from common.sql import PostgresHook +from database.batched_update.constants import DAG_ID as BATCHED_UPDATE_DAG_ID + + +DAG_ID = "decode_and_deduplicate_image_tags" + +HAS_RAW_ESCAPED_UNICODE = ( + r'(@.name like_regex "\\\\(x)([\\da-f]{2})|\\\\(u)([\\da-f]{4})" flag "i")' +) + + +@task +def ensure_ov_unistr( + postgres_conn_id: str = POSTGRES_CONN_ID, + task: AbstractOperator = None, +): + """ + Create a naïve implementation of Postgres 14+ ``unistr``. + + We are on Postgres 13 and have to do without ``unistr``. For all intents and purposes, + this implementation solves the problem for us. + + The ``ov`` prefix prevents clashing with the built-in should we upgrade. + """ + + postgres = PostgresHook( + postgres_conn_id=postgres_conn_id, + default_statement_timeout=PostgresHook.get_execution_timeout(task), + log_sql=True, + ) + + return postgres.run( + dedent( + """ + CREATE OR REPLACE FUNCTION ov_unistr (string text) + RETURNS text + AS $$ + return string.encode().decode("unicode_escape") if string else string + $$ LANGUAGE plpython3u; + """ + ) + ) + + +@dag( + dag_id=DAG_ID, + schedule=None, + start_date=datetime(2024, 6, 3), + tags=["database"], + doc_md=__doc__, + max_active_runs=1, + default_args=DAG_DEFAULT_ARGS, +) +def decode_and_deduplicate_image_tags(): + ensure_ov_unistr() >> TriggerDagRunOperator( + task_id="trigger_batched_update", + trigger_dag_id=BATCHED_UPDATE_DAG_ID, + wait_for_completion=True, + retries=0, + conf={ + "query_id": DAG_ID, + "table_name": "image", + # jsonb_path_query_first will return null if the first argument is null, + # and so is safe for tagless works + "select_query": dedent( + f""" + WHERE jsonb_path_query_first( + image.tags, + '$[*] ? {HAS_RAW_ESCAPED_UNICODE}' + ) IS NOT NULL + """ + ).strip(), + "update_query": dedent( + f""" + SET updated_on = NOW(), + tags = ( + SELECT jsonb_agg(deduplicated.tag) FROM ( + SELECT DISTINCT ON (all_tags.tag->'name', all_tags.tag->'provider') + all_tags.tag tag + FROM ( + SELECT + jsonb_array_elements( + separated_tags.no_escape || ( + SELECT jsonb_agg( + jsonb_set( + to_escape, + '{{name}}', + to_jsonb(ov_unistr(to_escape->>'name')) + ) + ) FROM jsonb_array_elements(separated_tags.needs_escape) AS to_escape + ) + ) tag + FROM ( + SELECT + jsonb_path_query_array(image.tags, '$[*] ? (!{HAS_RAW_ESCAPED_UNICODE})') no_escape, + jsonb_path_query_array(image.tags, '$[*] ? {HAS_RAW_ESCAPED_UNICODE}') needs_escape + ) AS separated_tags + ) AS all_tags + ) AS deduplicated + ) + """ + ).strip(), + "update_timeout": int(timedelta(hours=10).total_seconds()), + "dry_run": False, + }, + ) + + +decode_and_deduplicate_image_tags() diff --git a/catalog/dags/maintenance/trim_and_deduplicate_tags.py b/catalog/dags/maintenance/trim_and_deduplicate_tags.py index 23e971593fb..87976298b91 100644 --- a/catalog/dags/maintenance/trim_and_deduplicate_tags.py +++ b/catalog/dags/maintenance/trim_and_deduplicate_tags.py @@ -32,7 +32,6 @@ def trim_and_deduplicate_tags(): task_id=DAG_ID, trigger_dag_id=BATCHED_UPDATE_DAG_ID, wait_for_completion=True, - execution_timeout=timedelta(hours=5), max_active_tis_per_dag=2, map_index_template="""{{ task.conf['table_name'] }}""", retries=0, @@ -56,7 +55,7 @@ def trim_and_deduplicate_tags(): ) ), "update_query": ( - "SET updated_on = now(), " + "SET updated_on = NOW(), " + dedent( f""" tags = ( @@ -78,6 +77,7 @@ def trim_and_deduplicate_tags(): """ ) ), + "update_timeout": int(timedelta(hours=5).total_seconds()), "dry_run": False, } for media_type in MEDIA_TYPES diff --git a/documentation/catalog/reference/DAGs.md b/documentation/catalog/reference/DAGs.md index 46a21ce6760..79545b547af 100644 --- a/documentation/catalog/reference/DAGs.md +++ b/documentation/catalog/reference/DAGs.md @@ -48,6 +48,7 @@ The following are DAGs grouped by their primary tag: | DAG ID | Schedule Interval | | -------------------------------------------------------------------------------------- | ----------------- | | [`batched_update`](#batched_update) | `None` | +| [`decode_and_deduplicate_image_tags`](#decode_and_deduplicate_media_type_tags) | `None` | | [`delete_records`](#delete_records) | `None` | | [`recreate_full_staging_index`](#recreate_full_staging_index) | `None` | | [`recreate_audio_popularity_calculation`](#recreate_media_type_popularity_calculation) | `None` | @@ -144,6 +145,7 @@ The following is documentation associated with each DAG (where available): 1. [`create_new_production_es_index`](#create_new_environment_es_index) 1. [`create_new_staging_es_index`](#create_new_environment_es_index) 1. [`create_proportional_by_source_staging_index`](#create_proportional_by_source_staging_index) +1. [`decode_and_deduplicate_image_tags`](#decode_and_deduplicate_media_type_tags) 1. [`delete_records`](#delete_records) 1. [`europeana_workflow`](#europeana_workflow) 1. [`finnish_museums_workflow`](#finnish_museums_workflow) @@ -575,6 +577,21 @@ However, it will fail immediately if any of the DAGs tagged as part of the ---- +### `decode_and_deduplicate_{media_type}_tags` + +See the issue for context and motivation: +https://github.com/WordPress/openverse/issues/4452 + +This DAG triggers a run of the batched update DAG. It generates a new list of +tags by trimming all existing tags and re-inserting only the distinct tags of +the resulting list of tags. + +Only records before the CC Search -> Openverse transition are affected. As such, +because all audio records are dated after that transition, we only need to scan +images. + +---- + ### `delete_records` #### Delete Records DAG diff --git a/sample_data/sample_image.csv b/sample_data/sample_image.csv index 32295ae7724..7e8bb900cd6 100644 --- a/sample_data/sample_image.csv +++ b/sample_data/sample_image.csv @@ -1,7 +1,7 @@ identifier,created_on,updated_on,ingestion_type,provider,source,foreign_identifier,foreign_landing_url,url,thumbnail,width,height,filesize,license,license_version,creator,creator_url,title,meta_data,tags,watermarked,last_synced_with_source,removed_from_source,filetype,category,standardized_popularity 0e3315c5-3328-4a99-80ab-567ac32f685f,2022-12-21 17:29:54.000000+00,2022-12-21 17:29:54.000000+00,provider_api,flickr,flickr,51745822704,https://www.flickr.com/photos/54633257@N04/51745822704,https://live.staticflickr.com/65535/51745822704_ae97226e20_b.jpg,https://live.staticflickr.com/65535/51745822704_ae97226e20_m.jpg,433,1024,97150,by-nc-sa,2.0,mexicofist3,https://www.flickr.com/photos/54633257@N04,my dress is not very short,"{""views"": ""5991"", ""pub_date"": ""1639443671"", ""date_taken"": ""2021-07-20 18:19:25"", ""description"": ""dressed"", ""license_url"": ""https://creativecommons.org/licenses/by-nc-sa/2.0/"", ""raw_license_url"": null}",,f,2021-12-15 20:49:19.976193+00,f,jpg,photograph, b840de61-fb9d-4ec5-9572-8d778875869f,2022-03-08 14:32:25.000000+00,2022-03-08 14:32:25.000000+00,provider_api,flickr,flickr,51745349583,https://www.flickr.com/photos/129684398@N07/51745349583,https://live.staticflickr.com/65535/51745349583_cebf1fa6e0_b.jpg,https://live.staticflickr.com/65535/51745349583_cebf1fa6e0_m.jpg,1024,713,171956,by-nc,2.0,clairetresse,https://www.flickr.com/photos/129684398@N07,Barranco de las bodegas,"{""views"": ""4156"", ""pub_date"": ""1639440673"", ""date_taken"": ""2021-10-28 10:54:43"", ""description"": ""Descente dans un des Barranco, descente prudente parce que ça glisse. L’érosion des sols est extrêmement puissante, les reliefs sont sculptés par les forces de la nature telles que le vente les pluies . Descent in one of the Barranco, careful descent because it slips. The erosion of the soil is extremely powerful, the reliefs are sculpted by the forces of nature such as the rains."", ""license_url"": ""https://creativecommons.org/licenses/by-nc/2.0/"", ""raw_license_url"": null}","[{""name"": ""bardenas"", ""provider"": ""flickr""}, {""name"": ""barranco"", ""provider"": ""flickr""}, {""name"": ""desert"", ""provider"": ""flickr""}, {""name"": ""espagne"", ""provider"": ""flickr""}, {""name"": ""glaise"", ""provider"": ""flickr""}, {""name"": ""navarre"", ""provider"": ""flickr""}, {""name"": ""pyrennees"", ""provider"": ""flickr""}, {""name"": ""ravin"", ""provider"": ""flickr""}, {""name"": ""roches"", ""provider"": ""flickr""}, {""name"": ""water"", ""provider"": ""flickr""}, {""name"": ""lake"", ""provider"": ""machine_example"", ""accuracy"": 0.95}]",f,2021-12-15 20:49:19.976193+00,f,jpg,photograph, -aeba0547-61da-42ee-b561-27c8fc817d5a,2022-07-16 05:51:03.000000+00,2022-07-16 05:51:03.000000+00,provider_api,flickr,flickr,51745788239,https://www.flickr.com/photos/88123769@N02/51745788239,https://live.staticflickr.com/65535/51745788239_b645ce02fe_b.jpg,https://live.staticflickr.com/65535/51745788239_b645ce02fe_m.jpg,1024,602,281512,pdm,1.0,Bernard Spragg,https://www.flickr.com/photos/88123769@N02,Alone on the prairie.,"{""views"": ""1779"", ""pub_date"": ""1639441826"", ""date_taken"": ""2017-09-26 11:39:16"", ""description"": ""The Canadian Prairies (usually referred to as simply the Prairies in Canada) is a region in Western Canada. It includes the Canadian portion of the Great Plains and the Prairie Provinces, namely Alberta, Saskatchewan, and Manitoba.These provinces are partially covered by grasslands, plains, and lowlands, mostly in the southern regions."", ""license_url"": ""https://creativecommons.org/publicdomain/mark/1.0/"", ""raw_license_url"": null}","[{""name"": ""alberta"", ""provider"": ""flickr""}, {""name"": ""alone"", ""provider"": ""flickr""}, {""name"": ""canada"", ""provider"": ""flickr""}, {""name"": ""evening"", ""provider"": ""flickr""}, {""name"": ""house"", ""provider"": ""flickr""}, {""name"": ""landscape"", ""provider"": ""flickr""}, {""name"": ""lumixfz1000"", ""provider"": ""flickr""}, {""name"": ""old"", ""provider"": ""flickr""}, {""name"": ""outside"", ""provider"": ""flickr""}, {""name"": ""prairie"", ""provider"": ""flickr""}, {""name"": ""rural"", ""provider"": ""flickr""}, {""name"": ""scenery"", ""provider"": ""flickr""}, {""name"": ""sky"", ""provider"": ""flickr""}, {""name"": ""travel"", ""provider"": ""flickr""}]",f,2021-12-15 20:49:19.976193+00,f,jpg,photograph, +aeba0547-61da-42ee-b561-27c8fc817d5a,2022-07-16 05:51:03.000000+00,2022-07-16 05:51:03.000000+00,provider_api,flickr,flickr,51745788239,https://www.flickr.com/photos/88123769@N02/51745788239,https://live.staticflickr.com/65535/51745788239_b645ce02fe_b.jpg,https://live.staticflickr.com/65535/51745788239_b645ce02fe_m.jpg,1024,602,281512,pdm,1.0,Bernard Spragg,https://www.flickr.com/photos/88123769@N02,Alone on the prairie.,"{""views"": ""1779"", ""pub_date"": ""1639441826"", ""date_taken"": ""2017-09-26 11:39:16"", ""description"": ""The Canadian Prairies (usually referred to as simply the Prairies in Canada) is a region in Western Canada. It includes the Canadian portion of the Great Plains and the Prairie Provinces, namely Alberta, Saskatchewan, and Manitoba.These provinces are partially covered by grasslands, plains, and lowlands, mostly in the southern regions."", ""license_url"": ""https://creativecommons.org/publicdomain/mark/1.0/"", ""raw_license_url"": null}","[{""name"": ""alberta"", ""provider"": ""flickr""}, {""name"": ""alone"", ""provider"": ""flickr""}, {""name"": ""ciudaddelassiencias"", ""provider"": ""flickr""}, {""name"": ""muséo"", ""provider"": ""flickr""}, {""name"": ""muséo"", ""provider"": ""recognition"", ""accuracy"": 0.96}, {""name"": ""uploaded by me"", ""provider"": ""flickr""}, {}, {""name"": ""unknown"", ""provider"": ""recognition"", ""accuracy"": 0.86}, {""name"": ""mus\\xe9o"", ""provider"": ""flickr""}, {""name"": ""mus\\u00e9o"", ""provider"": ""flickr""}, {""name"": ""musu00e9o"", ""provider"": ""flickr""}, {""name"": ""mus\\u00e9o"", ""provider"": ""flickr""}, {""name"": ""mus\\u00E9o"", ""provider"": ""flickr""}, {""name"": ""canada"", ""provider"": ""flickr""}, {""name"": ""evening"", ""provider"": ""flickr""}, {""name"": ""house"", ""provider"": ""flickr""}, {""name"": ""landscape"", ""provider"": ""flickr""}, {""name"": ""lumixfz1000"", ""provider"": ""flickr""}, {""name"": ""old"", ""provider"": ""flickr""}, {""name"": ""outside"", ""provider"": ""flickr""}, {""name"": ""prairie"", ""provider"": ""flickr""}, {""name"": ""rural"", ""provider"": ""flickr""}, {""name"": ""scenery"", ""provider"": ""flickr""}, {""name"": ""sky"", ""provider"": ""flickr""}, {""name"": ""travel"", ""provider"": ""flickr""}]",f,2021-12-15 20:49:19.976193+00,f,jpg,photograph, 3c98150c-51a8-4175-a47f-acef10e784f7,2022-06-10 09:14:13.000000+00,2022-06-10 09:14:13.000000+00,provider_api,flickr,flickr,51747927224,https://www.flickr.com/photos/151325871@N07/51747927224,https://live.staticflickr.com/65535/51747927224_3ca7ac2e93.jpg,https://live.staticflickr.com/65535/51747927224_3ca7ac2e93_m.jpg,318,500,53633,cc0,1.0,lyndawaybi3,https://www.flickr.com/photos/151325871@N07,Naughty Little Elf,"{""views"": ""1342"", ""pub_date"": ""1639526583"", ""date_taken"": ""2021-12-14 16:02:55"", ""license_url"": ""https://creativecommons.org/publicdomain/zero/1.0/""}","[{""name"": ""babe"", ""provider"": ""flickr""}, {""name"": ""bi"", ""provider"": ""flickr""}, {""name"": ""brunette"", ""provider"": ""flickr""}, {""name"": ""chick"", ""provider"": ""flickr""}, {""name"": ""christmas"", ""provider"": ""flickr""}, {""name"": ""dress"", ""provider"": ""flickr""}, {""name"": ""elf"", ""provider"": ""flickr""}, {""name"": ""great"", ""provider"": ""flickr""}, {""name"": ""hot"", ""provider"": ""flickr""}, {""name"": ""hotwife"", ""provider"": ""flickr""}, {""name"": ""leggings"", ""provider"": ""flickr""}, {""name"": ""legs"", ""provider"": ""flickr""}, {""name"": ""lynda"", ""provider"": ""flickr""}, {""name"": ""married"", ""provider"": ""flickr""}, {""name"": ""milf"", ""provider"": ""flickr""}, {""name"": ""mini"", ""provider"": ""flickr""}, {""name"": ""mom"", ""provider"": ""flickr""}, {""name"": ""nylons"", ""provider"": ""flickr""}, {""name"": ""panyhose"", ""provider"": ""flickr""}, {""name"": ""season"", ""provider"": ""flickr""}, {""name"": ""sexy"", ""provider"": ""flickr""}, {""name"": ""short"", ""provider"": ""flickr""}, {""name"": ""skirt"", ""provider"": ""flickr""}, {""name"": ""stockings"", ""provider"": ""flickr""}, {""name"": ""sweater"", ""provider"": ""flickr""}, {""name"": ""wife"", ""provider"": ""flickr""}, {""name"": ""young"", ""provider"": ""flickr""}]",f,2021-12-15 22:19:02.971943+00,f,jpg,photograph, cdbd3bf6-1745-45bb-b399-61ee149cd58a,2022-12-28 15:41:34.000000+00,2022-12-28 15:41:34.000000+00,provider_api,flickr,flickr,51745389858,https://www.flickr.com/photos/126744325@N07/51745389858,https://live.staticflickr.com/65535/51745389858_c10358e1a3_b.jpg,https://live.staticflickr.com/65535/51745389858_c10358e1a3_m.jpg,1024,683,157497,by,2.0,Kristoffer Trolle,https://www.flickr.com/photos/126744325@N07,Train area in Copenhagen South / Tog område i Syd København,"{""views"": ""1337"", ""pub_date"": ""1639441947"", ""date_taken"": ""2021-07-14 23:49:46"", ""description"": ""This old train area in Copenhagen South will soon be transformed into a residential area. I love to go there and take photos. I used a Tiffen Black Pro Mist 1/4 filter for this photo, it gives that diffused highlights look, read more about it on my blog here . The photo is Creative Commons license: Use it for free. Keywords: train, tog, DSB, område, syd, København, south, Copenhagen, Danmark, Denmark, Fujifilm X-H1, Fujifilm XF 35mm f2 R WR, Tiffen Black Pro-Mist 1/4 filter"", ""license_url"": ""https://creativecommons.org/licenses/by/2.0/"", ""raw_license_url"": null}","[{""name"": ""copenhagen"", ""provider"": ""flickr""}, {""name"": ""danmark"", ""provider"": ""flickr""}, {""name"": ""denmark"", ""provider"": ""flickr""}, {""name"": ""dsb"", ""provider"": ""flickr""}, {""name"": ""fujifilmxf35mmf2rwr"", ""provider"": ""flickr""}, {""name"": ""fujifilmxh1"", ""provider"": ""flickr""}, {""name"": ""københavn"", ""provider"": ""flickr""}, {""name"": ""område"", ""provider"": ""flickr""}, {""name"": ""south"", ""provider"": ""flickr""}, {""name"": ""syd"", ""provider"": ""flickr""}, {""name"": ""tiffenblackpromist14filter"", ""provider"": ""flickr""}, {""name"": ""tog"", ""provider"": ""flickr""}, {""name"": ""train"", ""provider"": ""flickr""}]",f,2021-12-15 20:49:19.976193+00,f,jpg,photograph, a3583692-349d-4ab7-8649-dfb6ab25a9a6,2022-05-10 05:38:53.000000+00,2022-05-10 05:38:53.000000+00,provider_api,flickr,flickr,51748188420,https://www.flickr.com/photos/38959360@N07/51748188420,https://live.staticflickr.com/65535/51748188420_cac46bb1f3_b.jpg,https://live.staticflickr.com/65535/51748188420_cac46bb1f3_m.jpg,579,1024,131055,by,2.0,...Amame hasta con los dientes....,https://www.flickr.com/photos/38959360@N07,Say you'll see me again....,"{""views"": ""1188"", ""pub_date"": ""1639527704"", ""date_taken"": ""2021-12-14 19:21:36"", ""description"": ""♡ Click Here for Details., Credits ♡and More Photos ♡ ♡ My Facebook ♡ My Instagram"", ""license_url"": ""https://creativecommons.org/licenses/by/2.0/"", ""raw_license_url"": null}","[{""name"": ""altier"", ""provider"": ""flickr""}, {""name"": ""blog"", ""provider"": ""flickr""}, {""name"": ""blogger"", ""provider"": ""flickr""}, {""name"": ""cute"", ""provider"": ""flickr""}, {""name"": ""dress"", ""provider"": ""flickr""}, {""name"": ""event"", ""provider"": ""flickr""}, {""name"": ""gown"", ""provider"": ""flickr""}, {""name"": ""hollyhood"", ""provider"": ""flickr""}, {""name"": ""kaya"", ""provider"": ""flickr""}, {""name"": ""kimo"", ""provider"": ""flickr""}, {""name"": ""log"", ""provider"": ""flickr""}, {""name"": ""maitreya"", ""provider"": ""flickr""}, {""name"": ""nightdress"", ""provider"": ""flickr""}, {""name"": ""nitedress"", ""provider"": ""flickr""}, {""name"": ""thirsty"", ""provider"": ""flickr""}, {""name"": ""versa"", ""provider"": ""flickr""}]",f,2021-12-15 22:14:02.778385+00,f,jpg,photograph,