Fix failing test, improve import readability

src/cc_catalog_airflow/dags/provider_api_scripts/brooklyn_museum.py

@@ -1,8 +1,8 @@
 import os
 import logging
 import lxml.html as html
-from common.requester import DelayedRequester
-from common import ImageStore
+
+from common import DelayedRequester, ImageStore
 from util.loader import provider_details as prov
 
 logging.basicConfig(

src/cc_catalog_airflow/dags/provider_api_scripts/cleveland_museum_of_art.py

@@ -1,4 +1,5 @@
 import logging
+
 from common import DelayedRequester, ImageStore
 from util.loader import provider_details as prov
 

src/cc_catalog_airflow/dags/provider_api_scripts/finnish_museums.py

@@ -1,4 +1,5 @@
 import logging
+
 from common import DelayedRequester, ImageStore
 from util.loader import provider_details as prov
 

src/cc_catalog_airflow/dags/provider_api_scripts/museum_victoria.py

@@ -1,4 +1,5 @@
 import logging
+
 from common import DelayedRequester, ImageStore
 from util.loader import provider_details as prov
 

src/cc_catalog_airflow/dags/provider_api_scripts/raw_pixel.py

@@ -1,5 +1,3 @@
-from common.storage.image import ImageStore
-from common.requester import DelayedRequester
 import requests
 import logging
 from urllib.parse import urlparse, parse_qs

src/cc_catalog_airflow/dags/provider_api_scripts/test_raw_pixel.py

@@ -46,7 +46,7 @@ def test_get_image_list_correct():
     with patch.object(rwp, "_request_content", return_value=r):
         total, result = rwp._get_image_list()
         assert total == 22215
-        assert len(result) == 2
+        assert len(result) == 1
 
 
 def test_process_pages_giving_zero():
@@ -67,26 +67,20 @@ def test_get_foreign_id_url():
     r = _get_resource_json("total_images_example.json")
     with patch.object(rwp, "_request_content", return_value=r):
         result = rwp._get_image_list()[1]
-        expected_foreign_ids = [
-            2041320,
-            2041320,
-        ]
-        expected_foreign_urls = [
-            "https://www.rawpixel.com/image/2041320/world-map-drawn-oval-projection",
-            "https://www.rawpixel.com/image/2041320/world-map-drawn-oval-projection",
-        ]
-        for i in range(2):
-            foreign_id, foreign_url = rwp._get_foreign_id_url(image=result[0])
-            assert foreign_id == expected_foreign_ids[i]
-            assert foreign_url == expected_foreign_urls[i]
+        foreign_id, foreign_url = rwp._get_foreign_id_url(image=result[0])
+        assert foreign_id == 2041320
+        assert (
+            foreign_url
+            == "https://www.rawpixel.com/image/2041320/world-map-drawn-oval-projection"
+        )
 
 
 def test_get_image_properties():
     r = _get_resource_json("total_images_example.json")
     with patch.object(rwp, "_request_content", return_value=r):
         result = rwp._get_image_list()[1]
         img_url, width, height, thumbnail = rwp._get_image_properties(
-            image=result[0]
+            image=result[0], foreign_url=""
         )
         assert (
                 img_url
@@ -108,26 +102,6 @@ def test_get_image_properties():
                     "fc1d64a")
         )
 
-        img_url, width, height, thumbnail = rwp._get_image_properties(
-            image=result[1]
-        )
-        assert (
-                img_url
-                == ("https://images.rawpixel.com/image_png_1300/czNmcy1wcml"
-                    "2YXRlL3Jhd3BpeGVsX2ltYWdlcy93ZWJzaXRlX2NvbnRlbnQvZmw0OTk"
-                    "0MjIxMjA1MS1pbWFnZS1rcDR2eDlhNi5wbmc.png?s=jMjsC0i4AdUsn"
-                    "ZY91VWFxTyuHEAgLqjmCRIH7_b76R0")
-        )
-        assert width == '1300'
-        assert height == '731'
-        assert (
-                thumbnail
-                == ("https://images.rawpixel.com/image_png_400/czNmcy1wcml2YXR"
-                    "lL3Jhd3BpeGVsX2ltYWdlcy93ZWJzaXRlX2NvbnRlbnQvZmw0OTk0MjIxM"
-                    "jA1MS1pbWFnZS1rcDR2eDlhNi5wbmc.png?s=DVKwFVZT14d39M_xKMN8i"
-                    "hYLeaqv8wUoT4cCaWCuVm4")
-        )
-
 
 def test_get_title_owner():
     r = _get_resource_json("total_images_example.json")
@@ -142,22 +116,15 @@ def test_get_meta_data_given_pinterest_descr_is_present():
     r = _get_resource_json("total_images_example.json")
     with patch.object(rwp, "_request_content", return_value=r):
         result = rwp._get_image_list()[1]
-        expected_data = [
-            "Portolan atlas of the Mediterranean Sea, western Europe, and the "
-            "northwest coast of Africa: World map drawn on an oval projection"
+        meta_data = rwp._get_meta_data(image=result[0])
+        expected_descr_value = (
+            "Portolan atlas of the Mediterranean Sea, western Europe, and the"
+            " northwest coast of Africa: World map drawn on an oval projection"
             " (ca. 1590) by Joan Oliva. Original from Library of Congress. "
             "Digitally enhanced by rawpixel. | free image by rawpixel.com / "
-            "Library of Congress (Source)",
-            "From research to community support, Oak Ridge National "
-            "Laboratory enlists world-class science and staff in the fight "
-            "against COVID-19. Original public domain image from Flickr | "
-            "free image by rawpixel.com / United States Department of "
-            "Energy (source)",
-        ]
-        for i in range(2):
-            meta_data = rwp._get_meta_data(image=result[i])
-            expected_meta_data = {"description": expected_data[i]}
-            assert meta_data == expected_meta_data
+            "Library of Congress (Source)")
+        expected_meta_data = {"description": expected_descr_value}
+        assert meta_data == expected_meta_data
 
 
 def test_get_meta_data_given_no_pinterest_descr():
@@ -172,11 +139,6 @@ def test_get_tags():
     r = _get_resource_json("total_images_example.json")
     with patch.object(rwp, "_request_content", return_value=r):
         result = rwp._get_image_list()[1]
-        expected_data = [
-            {"tag_count": 47, "first_tag": "america"},
-            {"tag_count": 12, "first_tag": "coronavirus"},
-        ]
-        for i in range(2):
-            tags = rwp._get_tags(image=result[i])
-            assert len(tags) == expected_data[i]["tag_count"]
-            assert tags[0] == expected_data[i]["first_tag"]
+        tags = rwp._get_tags(image=result[0])
+        assert len(tags) == 47
+        assert tags[0] == "america"

src/cc_catalog_airflow/dags/provider_api_scripts/walters_art_museum.py

@@ -12,6 +12,7 @@
 
 import os
 import logging
+
 from common import DelayedRequester, ImageStore
 from util.loader import provider_details as prov