From 22866dcca17eefa3c386237094e7bccc2bfc16b5 Mon Sep 17 00:00:00 2001 From: Matthias Wright Date: Fri, 24 May 2024 22:11:16 +0800 Subject: [PATCH] feat(origin-ipfs): support different request url formats --- core/fetcher/src/tests.rs | 3 ++- core/origin-ipfs/src/config.rs | 29 +++++++++++++++++++++++++++++ core/origin-ipfs/src/origin_ipfs.rs | 6 +----- core/origin-ipfs/src/tests.rs | 6 +++++- 4 files changed, 37 insertions(+), 7 deletions(-) diff --git a/core/fetcher/src/tests.rs b/core/fetcher/src/tests.rs index 16232b944..71c556a58 100644 --- a/core/fetcher/src/tests.rs +++ b/core/fetcher/src/tests.rs @@ -21,7 +21,7 @@ use lightning_interfaces::types::{ }; use lightning_notifier::Notifier; use lightning_origin_demuxer::{Config as DemuxerOriginConfig, OriginDemuxer}; -use lightning_origin_ipfs::config::{Gateway, Protocol}; +use lightning_origin_ipfs::config::{Gateway, Protocol, RequestFormat}; use lightning_origin_ipfs::Config as IPFSOriginConfig; use lightning_pool::{Config as PoolConfig, PoolProvider}; use lightning_rep_collector::aggregator::ReputationAggregator; @@ -142,6 +142,7 @@ async fn get_fetchers( "127.0.0.1:{}", gateway_port_offset + i as u16 ), + request_format: RequestFormat::CidLast, }], gateway_timeout: Duration::from_millis(5000), }, diff --git a/core/origin-ipfs/src/config.rs b/core/origin-ipfs/src/config.rs index c7c6afd77..23ad2b8bf 100644 --- a/core/origin-ipfs/src/config.rs +++ b/core/origin-ipfs/src/config.rs @@ -1,5 +1,6 @@ use std::time::Duration; +use cid::Cid; use serde::{Deserialize, Serialize}; #[derive(Serialize, Deserialize)] @@ -15,27 +16,42 @@ pub enum Protocol { Https, } +#[derive(Serialize, Deserialize, Debug)] +pub enum RequestFormat { + CidFirst, + CidLast, +} + #[derive(Serialize, Deserialize, Debug)] pub struct Gateway { pub protocol: Protocol, pub authority: String, + pub request_format: RequestFormat, } impl Default for Config { fn default() -> Self { Self { gateways: vec![ + Gateway { + protocol: Protocol::Https, + authority: "ipfs.w3s.link".to_string(), + request_format: RequestFormat::CidFirst, + }, Gateway { protocol: Protocol::Https, authority: "fleek.ipfs.io".to_string(), + request_format: RequestFormat::CidLast, }, Gateway { protocol: Protocol::Https, authority: "ipfs.io".to_string(), + request_format: RequestFormat::CidLast, }, Gateway { protocol: Protocol::Https, authority: "ipfs.runfission.com".to_string(), + request_format: RequestFormat::CidLast, }, ], gateway_timeout: Duration::from_millis(5000), @@ -43,6 +59,19 @@ impl Default for Config { } } +impl Gateway { + pub fn build_request(&self, cid: Cid) -> String { + match self.request_format { + RequestFormat::CidFirst => { + format!("{}://{cid}.{}", self.protocol.as_str(), self.authority) + }, + RequestFormat::CidLast => { + format!("{}://{}/ipfs/{cid}", self.protocol.as_str(), self.authority) + }, + } + } +} + impl Protocol { pub fn as_str(&self) -> &str { match self { diff --git a/core/origin-ipfs/src/origin_ipfs.rs b/core/origin-ipfs/src/origin_ipfs.rs index 837630d3b..72e78f816 100644 --- a/core/origin-ipfs/src/origin_ipfs.rs +++ b/core/origin-ipfs/src/origin_ipfs.rs @@ -156,11 +156,7 @@ impl IPFSOrigin { pub async fn fetch(&self, uri: &[u8]) -> Result { let requested_cid = Cid::try_from(uri).with_context(|| "Failed to parse uri into cid")?; for gateway in self.gateways.iter() { - let url = Uri::builder() - .scheme(gateway.protocol.as_str()) - .authority(gateway.authority.as_str()) - .path_and_query(format!("/ipfs/{requested_cid}")) - .build()?; + let url: Uri = gateway.build_request(requested_cid).parse()?; let req = Request::builder() .uri(url) diff --git a/core/origin-ipfs/src/tests.rs b/core/origin-ipfs/src/tests.rs index e0c308ac6..181adb2ab 100644 --- a/core/origin-ipfs/src/tests.rs +++ b/core/origin-ipfs/src/tests.rs @@ -18,7 +18,7 @@ use lightning_test_utils::json_config::JsonConfigProvider; use lightning_test_utils::keys::EphemeralKeystore; use lightning_test_utils::server::spawn_server; -use crate::config::{Config, Gateway, Protocol}; +use crate::config::{Config, Gateway, Protocol, RequestFormat}; use crate::IPFSOrigin; partial!(TestBinding { @@ -170,6 +170,7 @@ async fn test_origin_dag_pb() { config.gateways = vec![Gateway { protocol: Protocol::Http, authority: "127.0.0.1:30100".to_string(), + request_format: RequestFormat::CidLast, }]; let ipfs_origin = IPFSOrigin::::new(config, state.blockstore().clone()).unwrap(); @@ -210,6 +211,7 @@ async fn test_origin_bbb_dag_pb() { config.gateways = vec![Gateway { protocol: Protocol::Http, authority: "127.0.0.1:30200".to_string(), + request_format: RequestFormat::CidLast, }]; let ipfs_origin = IPFSOrigin::::new(config, state.blockstore().clone()).unwrap(); @@ -250,6 +252,7 @@ async fn test_origin_raw() { config.gateways = vec![Gateway { protocol: Protocol::Http, authority: "127.0.0.1:30201".to_string(), + request_format: RequestFormat::CidLast, }]; let ipfs_origin = IPFSOrigin::::new(config, state.blockstore().clone()).unwrap(); @@ -290,6 +293,7 @@ async fn test_origin_bbb_dag_pb_and_raw() { config.gateways = vec![Gateway { protocol: Protocol::Http, authority: "127.0.0.1:30202".to_string(), + request_format: RequestFormat::CidLast, }]; let ipfs_origin = IPFSOrigin::::new(config, state.blockstore().clone()).unwrap();