Merge pull request #84 from ourzora/fix-to-adapter

Fix to adapter

offchain/metadata/adapters/arweave.py

@@ -1,5 +1,7 @@
 import random
 from typing import Optional
+
+import httpx
 from requests import PreparedRequest, Response
 from urllib3.util import parse_url
 
@@ -27,16 +29,21 @@ def __init__(
         *args,
         **kwargs,
     ):
-
         self.host_prefixes = host_prefixes or ["https://arweave.net/"]
 
-        assert all([g.endswith("/") for g in self.host_prefixes]), "gateways should have trailing slashes"
+        assert all(
+            [g.endswith("/") for g in self.host_prefixes]
+        ), "gateways should have trailing slashes"
 
         self.key = key
         self.secret = secret
         self.timeout = timeout
         super().__init__(*args, **kwargs)
 
+    async def gen_send(self, url: str, *args, **kwargs) -> httpx.Response:
+        # TODO(Isabella): implement here
+        raise NotImplementedError
+
     def send(self, request: PreparedRequest, *args, **kwargs) -> Response:
         """Format and send request to ARWeave host.
 

offchain/metadata/adapters/base_adapter.py

@@ -1,10 +1,9 @@
 from dataclasses import dataclass, field
 from typing import Optional, Type, Union
 
-from requests.adapters import (
-    BaseAdapter as RequestsBaseAdapter,
-    HTTPAdapter as RequestsHTTPAdapter,
-)
+import httpx
+from requests.adapters import BaseAdapter as RequestsBaseAdapter
+from requests.adapters import HTTPAdapter as RequestsHTTPAdapter
 from urllib3.util.retry import Retry
 
 
@@ -14,6 +13,9 @@ class BaseAdapter(RequestsBaseAdapter):
     def __init__(self, *args, **kwargs):
         super().__init__()
 
+    async def gen_send(self, url: str, *args, **kwargs) -> httpx.Response:
+        raise NotImplementedError
+
 
 class HTTPAdapter(RequestsHTTPAdapter):
     """HTTP Adapter inheriting from requests HTTPAdapter"""
@@ -29,6 +31,10 @@ def __init__(
     ) -> None:
         super().__init__(pool_connections, pool_maxsize, max_retries, pool_block)
 
+    async def gen_send(self, url: str, *args, **kwargs) -> httpx.Response:
+        # TODO(Isabella): implement here
+        raise NotImplementedError
+
 
 Adapter = Union[BaseAdapter, HTTPAdapter]
 

offchain/metadata/adapters/data_uri.py

@@ -1,17 +1,41 @@
+import base64
 from urllib.request import urlopen
+
+import httpx
 from requests import PreparedRequest, Response
 
 from offchain.metadata.adapters.base_adapter import BaseAdapter
 from offchain.metadata.registries.adapter_registry import AdapterRegistry
 
 
+def decode_data_url(data_url):
+    data_parts = data_url.split(",")
+    data = data_parts[1]
+
+    if ";base64" in data_parts[0]:
+        # media_type = data_parts[0].replace(";base64", "")
+        decoded_data = base64.b64decode(data)
+        decoded_text = decoded_data.decode("utf-8")
+        return decoded_text
+
+    return None
+
+
 @AdapterRegistry.register
 class DataURIAdapter(BaseAdapter):
     """Provides an interface for Requests sessions to handle data uris."""
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
 
+    async def gen_send(self, url: str, *args, **kwargs):
+        response = httpx.Response(
+            status_code=200,
+            text=decode_data_url(url),
+            request=httpx.Request(method="GET", url=url),
+        )
+        return response
+
     def send(self, request: PreparedRequest, *args, **kwargs):
         """Handle data uri request.
 

offchain/metadata/adapters/ipfs.py

@@ -1,6 +1,8 @@
 import random
-from requests import PreparedRequest, Response
 from typing import Optional
+
+import httpx
+from requests import PreparedRequest, Response
 from urllib3.util import parse_url
 
 from offchain.metadata.adapters.base_adapter import HTTPAdapter
@@ -69,10 +71,11 @@ def __init__(
         *args,
         **kwargs,
     ):
-
         self.host_prefixes = host_prefixes or ["https://gateway.pinata.cloud/ipfs/"]
 
-        assert all([g.endswith("/") for g in self.host_prefixes]), "gateways should have trailing slashes"
+        assert all(
+            [g.endswith("/") for g in self.host_prefixes]
+        ), "gateways should have trailing slashes"
 
         self.key = key
         self.secret = secret
@@ -93,6 +96,10 @@ def make_request_url(self, request_url: str, gateway: Optional[str] = None) -> s
         gateway = gateway or random.choice(self.host_prefixes)
         return build_request_url(gateway=gateway, request_url=request_url)
 
+    async def gen_send(self, url: str, *args, **kwargs) -> httpx.Response:
+        # TODO(Isabella): implement here
+        raise NotImplementedError
+
     def send(self, request: PreparedRequest, *args, **kwargs) -> Response:
         """For IPFS hashes, query pinata cloud gateway
 

offchain/metadata/fetchers/metadata_fetcher.py

@@ -1,12 +1,13 @@
 import cgi
-import requests
+from typing import Tuple, Union
+
 import aiohttp
 import httpx
-from typing import Tuple, Union
+import requests
 
+from offchain.logger.logging import logger
 from offchain.metadata.adapters.base_adapter import Adapter, AdapterConfig
 from offchain.metadata.fetchers.base_fetcher import BaseFetcher
-from offchain.logger.logging import logger
 from offchain.metadata.registries.fetcher_registry import FetcherRegistry
 
 
@@ -60,8 +61,16 @@ def _head(self, uri: str):
 
     def _get(self, uri: str):
         return self.sess.get(uri, timeout=self.timeout, allow_redirects=True)
-
-    async def gen_get(self, uri: str):
+
+    async def gen(self, uri: str) -> httpx.Response:
+        from offchain.metadata.pipelines.metadata_pipeline import (
+            DEFAULT_ADAPTER_CONFIGS,
+        )
+
+        for adapter_config in DEFAULT_ADAPTER_CONFIGS:
+            if any(uri.startswith(prefix) for prefix in adapter_config.mount_prefixes):
+                adapter = adapter_config.adapter_cls(**adapter_config.kwargs)
+                return await adapter.gen_send(url=uri, timeout=self.timeout)
         return await self.async_sess.get(uri, timeout=self.timeout)
 
     def fetch_mime_type_and_size(self, uri: str) -> Tuple[str, int]:
@@ -87,7 +96,9 @@ def fetch_mime_type_and_size(self, uri: str) -> Tuple[str, int]:
 
             return content_type, size
         except Exception as e:
-            logger.error(f"Failed to fetch content-type and size from uri {uri}. Error: {e}")
+            logger.error(
+                f"Failed to fetch content-type and size from uri {uri}. Error: {e}"
+            )
             raise
 
     def fetch_content(self, uri: str) -> Union[dict, str]:
@@ -120,13 +131,12 @@ async def gen_fetch_content(self, uri: str) -> Union[dict, str]:
             Union[dict, str]: content fetched from uri
         """
         try:
-            res = await self.gen_get(uri)
-            async with res:
-                res.raise_for_status()
-                if res.text.startswith("{"):
-                    return res.json()
-                else:
-                    return res.text
+            res = await self.gen(uri)
+            res.raise_for_status()
+            if res.text.startswith("{"):
+                return res.json()
+            else:
+                return res.text
 
         except Exception as e:
-            raise Exception(f"Don't know how to fetch metadata for {uri=}. {str(e)}")
+            raise Exception(f"Don't know how to fetch metadata for {uri=}. {str(e)}")

pyproject.toml

@@ -16,6 +16,9 @@ web3 = "^5.30.0"
 tenacity = "^8.0.1"
 aiohttp = "^3.8.5"
 httpx = "^0.24.1"
+types-requests = "^2.31.0.2"
+pytest-asyncio = "^0.21.1"
+pytest-async = "^0.1.1"
 
 [tool.poetry.dev-dependencies]
 pre-commit = "^2.20.0"
@@ -26,6 +29,9 @@ mkdocs-material = "^8.4.2"
 mkdocstrings = "^0.19.0"
 mkdocstrings-python = "^0.7.1"
 
+[tool.poetry.group.dev.dependencies]
+pytest-asyncio = "^0.21.1"
+
 [build-system]
 requires = ["poetry-core>=1.0.0"]
 build-backend = "poetry.core.masonry.api"