feat(origin-ipfs): support different request url formats

fleek-network · May 24, 2024 · 22866dc · 22866dc
1 parent bb49391
commit 22866dc
Show file tree

Hide file tree

Showing 4 changed files with 37 additions and 7 deletions.
diff --git a/core/fetcher/src/tests.rs b/core/fetcher/src/tests.rs
@@ -21,7 +21,7 @@ use lightning_interfaces::types::{
 };
 use lightning_notifier::Notifier;
 use lightning_origin_demuxer::{Config as DemuxerOriginConfig, OriginDemuxer};
-use lightning_origin_ipfs::config::{Gateway, Protocol};
+use lightning_origin_ipfs::config::{Gateway, Protocol, RequestFormat};
 use lightning_origin_ipfs::Config as IPFSOriginConfig;
 use lightning_pool::{Config as PoolConfig, PoolProvider};
 use lightning_rep_collector::aggregator::ReputationAggregator;
@@ -142,6 +142,7 @@ async fn get_fetchers(
                                         "127.0.0.1:{}",
                                         gateway_port_offset + i as u16
                                     ),
+                                    request_format: RequestFormat::CidLast,
                                 }],
                                 gateway_timeout: Duration::from_millis(5000),
                             },

diff --git a/core/origin-ipfs/src/config.rs b/core/origin-ipfs/src/config.rs
@@ -1,5 +1,6 @@
 use std::time::Duration;
 
+use cid::Cid;
 use serde::{Deserialize, Serialize};
 
 #[derive(Serialize, Deserialize)]
@@ -15,34 +16,62 @@ pub enum Protocol {
     Https,
 }
 
+#[derive(Serialize, Deserialize, Debug)]
+pub enum RequestFormat {
+    CidFirst,
+    CidLast,
+}
+
 #[derive(Serialize, Deserialize, Debug)]
 pub struct Gateway {
     pub protocol: Protocol,
     pub authority: String,
+    pub request_format: RequestFormat,
 }
 
 impl Default for Config {
     fn default() -> Self {
         Self {
             gateways: vec![
+                Gateway {
+                    protocol: Protocol::Https,
+                    authority: "ipfs.w3s.link".to_string(),
+                    request_format: RequestFormat::CidFirst,
+                },
                 Gateway {
                     protocol: Protocol::Https,
                     authority: "fleek.ipfs.io".to_string(),
+                    request_format: RequestFormat::CidLast,
                 },
                 Gateway {
                     protocol: Protocol::Https,
                     authority: "ipfs.io".to_string(),
+                    request_format: RequestFormat::CidLast,
                 },
                 Gateway {
                     protocol: Protocol::Https,
                     authority: "ipfs.runfission.com".to_string(),
+                    request_format: RequestFormat::CidLast,
                 },
             ],
             gateway_timeout: Duration::from_millis(5000),
         }
     }
 }
 
+impl Gateway {
+    pub fn build_request(&self, cid: Cid) -> String {
+        match self.request_format {
+            RequestFormat::CidFirst => {
+                format!("{}://{cid}.{}", self.protocol.as_str(), self.authority)
+            },
+            RequestFormat::CidLast => {
+                format!("{}://{}/ipfs/{cid}", self.protocol.as_str(), self.authority)
+            },
+        }
+    }
+}
+
 impl Protocol {
     pub fn as_str(&self) -> &str {
         match self {

diff --git a/core/origin-ipfs/src/origin_ipfs.rs b/core/origin-ipfs/src/origin_ipfs.rs
@@ -156,11 +156,7 @@ impl<C: Collection> IPFSOrigin<C> {
     pub async fn fetch(&self, uri: &[u8]) -> Result<Blake3Hash> {
         let requested_cid = Cid::try_from(uri).with_context(|| "Failed to parse uri into cid")?;
         for gateway in self.gateways.iter() {
-            let url = Uri::builder()
-                .scheme(gateway.protocol.as_str())
-                .authority(gateway.authority.as_str())
-                .path_and_query(format!("/ipfs/{requested_cid}"))
-                .build()?;
+            let url: Uri = gateway.build_request(requested_cid).parse()?;
 
             let req = Request::builder()
                 .uri(url)

diff --git a/core/origin-ipfs/src/tests.rs b/core/origin-ipfs/src/tests.rs
@@ -18,7 +18,7 @@ use lightning_test_utils::json_config::JsonConfigProvider;
 use lightning_test_utils::keys::EphemeralKeystore;
 use lightning_test_utils::server::spawn_server;
 
-use crate::config::{Config, Gateway, Protocol};
+use crate::config::{Config, Gateway, Protocol, RequestFormat};
 use crate::IPFSOrigin;
 
 partial!(TestBinding {
@@ -170,6 +170,7 @@ async fn test_origin_dag_pb() {
         config.gateways = vec![Gateway {
             protocol: Protocol::Http,
             authority: "127.0.0.1:30100".to_string(),
+            request_format: RequestFormat::CidLast,
         }];
         let ipfs_origin =
             IPFSOrigin::<TestBinding>::new(config, state.blockstore().clone()).unwrap();
@@ -210,6 +211,7 @@ async fn test_origin_bbb_dag_pb() {
         config.gateways = vec![Gateway {
             protocol: Protocol::Http,
             authority: "127.0.0.1:30200".to_string(),
+            request_format: RequestFormat::CidLast,
         }];
         let ipfs_origin =
             IPFSOrigin::<TestBinding>::new(config, state.blockstore().clone()).unwrap();
@@ -250,6 +252,7 @@ async fn test_origin_raw() {
         config.gateways = vec![Gateway {
             protocol: Protocol::Http,
             authority: "127.0.0.1:30201".to_string(),
+            request_format: RequestFormat::CidLast,
         }];
         let ipfs_origin =
             IPFSOrigin::<TestBinding>::new(config, state.blockstore().clone()).unwrap();
@@ -290,6 +293,7 @@ async fn test_origin_bbb_dag_pb_and_raw() {
         config.gateways = vec![Gateway {
             protocol: Protocol::Http,
             authority: "127.0.0.1:30202".to_string(),
+            request_format: RequestFormat::CidLast,
         }];
         let ipfs_origin =
             IPFSOrigin::<TestBinding>::new(config, state.blockstore().clone()).unwrap();