diff --git a/Cargo.toml b/Cargo.toml index 4215e1f..f69e500 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pmtiles" -version = "0.6.0" +version = "0.7.0" edition = "2021" authors = ["Luke Seelenbinder "] license = "MIT OR Apache-2.0" @@ -12,31 +12,39 @@ categories = ["science::geo"] [features] default = [] -http-async = ["dep:tokio", "dep:reqwest"] -s3-async-native = ["dep:tokio", "dep:rust-s3", "rust-s3/tokio-native-tls"] -s3-async-rustls = ["dep:tokio", "dep:rust-s3", "rust-s3/tokio-rustls-tls"] -mmap-async-tokio = ["dep:tokio", "dep:fmmap", "fmmap?/tokio-async"] +http-async = ["__async", "dep:reqwest"] +mmap-async-tokio = ["__async", "dep:fmmap", "fmmap?/tokio-async"] +s3-async-native = ["__async-s3"] +s3-async-rustls = ["__async-s3"] tilejson = ["dep:tilejson", "dep:serde", "dep:serde_json"] -# TODO: support other async libraries +# Forward some of the common features to reqwest dependency +reqwest-default = ["reqwest?/default"] +reqwest-native-tls = ["reqwest?/native-tls"] +reqwest-rustls-tls = ["reqwest?/rustls-tls"] +reqwest-rustls-tls-webpki-roots = ["reqwest?/rustls-tls-webpki-roots"] +reqwest-rustls-tls-native-roots = ["reqwest?/rustls-tls-native-roots"] + +# Internal features, do not use +__async = ["dep:tokio", "async-compression/tokio"] +__async-s3 = ["__async", "dep:rust-s3", "rust-s3?/tokio-native-tls"] [dependencies] # TODO: determine how we want to handle compression in async & sync environments -# TODO: tokio is always requested here, but the tokio dependency is optional below - maybe make it required? -async-compression = { version = "0.4", features = ["gzip", "zstd", "brotli", "tokio"] } +async-compression = { version = "0.4", features = ["gzip", "zstd", "brotli"] } async-recursion = "1" async-trait = "0.1" bytes = "1" fmmap = { version = "0.3", default-features = false, optional = true } hilbert_2d = "1" reqwest = { version = "0.11", default-features = false, optional = true } +rust-s3 = { version = "0.33.0", optional = true, default-features = false, features = ["fail-on-err"] } serde = { version = "1", optional = true } serde_json = { version = "1", optional = true } thiserror = "1" tilejson = { version = "0.4", optional = true } tokio = { version = "1", default-features = false, features = ["io-util"], optional = true } varint-rs = "2" -rust-s3 = { version = "0.33.0", optional = true, default-features = false, features = ["fail-on-err"] } [dev-dependencies] flate2 = "1" diff --git a/src/async_reader.rs b/src/async_reader.rs index 1126b46..a8ff6d3 100644 --- a/src/async_reader.rs +++ b/src/async_reader.rs @@ -2,35 +2,14 @@ // so any file larger than 4GB, or an untrusted file with bad data may crash. #![allow(clippy::cast_possible_truncation)] -#[cfg(feature = "mmap-async-tokio")] -use std::path::Path; - use async_recursion::async_recursion; use async_trait::async_trait; use bytes::Bytes; -#[cfg(feature = "http-async")] -use reqwest::{Client, IntoUrl}; -#[cfg(any( - feature = "http-async", - feature = "mmap-async-tokio", - feature = "s3-async-rustls", - feature = "s3-async-native" -))] +#[cfg(feature = "__async")] use tokio::io::AsyncReadExt; -#[cfg(feature = "http-async")] -use crate::backend::HttpBackend; -#[cfg(feature = "mmap-async-tokio")] -use crate::backend::MmapBackend; -#[cfg(any(feature = "s3-async-rustls", feature = "s3-async-native"))] -use crate::backend::S3Backend; use crate::cache::DirCacheResult; -#[cfg(any( - feature = "http-async", - feature = "mmap-async-tokio", - feature = "s3-async-native", - feature = "s3-async-rustls" -))] +#[cfg(feature = "__async")] use crate::cache::{DirectoryCache, NoCache}; use crate::directory::{DirEntry, Directory}; use crate::error::{PmtError, PmtResult}; @@ -227,80 +206,6 @@ impl AsyncPmTile } } -#[cfg(feature = "http-async")] -impl AsyncPmTilesReader { - /// Creates a new `PMTiles` reader from a URL using the Reqwest backend. - /// - /// Fails if [url] does not exist or is an invalid archive. (Note: HTTP requests are made to validate it.) - pub async fn new_with_url(client: Client, url: U) -> PmtResult { - Self::new_with_cached_url(NoCache, client, url).await - } -} - -#[cfg(feature = "http-async")] -impl AsyncPmTilesReader { - /// Creates a new `PMTiles` reader with cache from a URL using the Reqwest backend. - /// - /// Fails if [url] does not exist or is an invalid archive. (Note: HTTP requests are made to validate it.) - pub async fn new_with_cached_url( - cache: C, - client: Client, - url: U, - ) -> PmtResult { - let backend = HttpBackend::try_from(client, url)?; - - Self::try_from_cached_source(backend, cache).await - } -} - -#[cfg(feature = "mmap-async-tokio")] -impl AsyncPmTilesReader { - /// Creates a new `PMTiles` reader from a file path using the async mmap backend. - /// - /// Fails if [p] does not exist or is an invalid archive. - pub async fn new_with_path>(path: P) -> PmtResult { - Self::new_with_cached_path(NoCache, path).await - } -} - -#[cfg(feature = "mmap-async-tokio")] -impl AsyncPmTilesReader { - /// Creates a new cached `PMTiles` reader from a file path using the async mmap backend. - /// - /// Fails if [p] does not exist or is an invalid archive. - pub async fn new_with_cached_path>(cache: C, path: P) -> PmtResult { - let backend = MmapBackend::try_from(path).await?; - - Self::try_from_cached_source(backend, cache).await - } -} - -#[cfg(any(feature = "s3-async-native", feature = "s3-async-rustls"))] -impl AsyncPmTilesReader { - /// Creates a new `PMTiles` reader from a URL using the Reqwest backend. - /// - /// Fails if [url] does not exist or is an invalid archive. (Note: HTTP requests are made to validate it.) - pub async fn new_with_bucket_path(bucket: s3::Bucket, path: String) -> PmtResult { - Self::new_with_cached_bucket_path(NoCache, bucket, path).await - } -} - -#[cfg(any(feature = "s3-async-native", feature = "s3-async-rustls"))] -impl AsyncPmTilesReader { - /// Creates a new `PMTiles` reader with cache from a URL using the Reqwest backend. - /// - /// Fails if [url] does not exist or is an invalid archive. (Note: HTTP requests are made to validate it.) - pub async fn new_with_cached_bucket_path( - cache: C, - bucket: s3::Bucket, - path: String, - ) -> PmtResult { - let backend = S3Backend::from(bucket, path); - - Self::try_from_cached_source(backend, cache).await - } -} - #[async_trait] pub trait AsyncBackend { /// Reads exactly `length` bytes starting at `offset` @@ -314,8 +219,8 @@ pub trait AsyncBackend { #[cfg(feature = "mmap-async-tokio")] mod tests { use super::AsyncPmTilesReader; - use crate::backend::MmapBackend; use crate::tests::{RASTER_FILE, VECTOR_FILE}; + use crate::MmapBackend; #[tokio::test] async fn open_sanity_check() { diff --git a/src/backend/mod.rs b/src/backend/mod.rs deleted file mode 100644 index c6341f2..0000000 --- a/src/backend/mod.rs +++ /dev/null @@ -1,17 +0,0 @@ -#[cfg(any(feature = "s3-async-native", feature = "s3-async-rustls"))] -mod s3; - -#[cfg(any(feature = "s3-async-native", feature = "s3-async-rustls"))] -pub use s3::S3Backend; - -#[cfg(feature = "http-async")] -mod http; - -#[cfg(feature = "http-async")] -pub use http::HttpBackend; - -#[cfg(feature = "mmap-async-tokio")] -mod mmap; - -#[cfg(feature = "mmap-async-tokio")] -pub use mmap::MmapBackend; diff --git a/src/backend/s3.rs b/src/backend/s3.rs deleted file mode 100644 index f13ba38..0000000 --- a/src/backend/s3.rs +++ /dev/null @@ -1,55 +0,0 @@ -use async_trait::async_trait; -use bytes::Bytes; -use s3::Bucket; - -use crate::{ - async_reader::AsyncBackend, - error::PmtError::{ResponseBodyTooLong, UnexpectedNumberOfBytesReturned}, -}; - -pub struct S3Backend { - bucket: Bucket, - pmtiles_path: String, -} - -impl S3Backend { - #[must_use] - pub fn from(bucket: Bucket, pmtiles_path: String) -> S3Backend { - Self { - bucket, - pmtiles_path, - } - } -} - -#[async_trait] -impl AsyncBackend for S3Backend { - async fn read_exact(&self, offset: usize, length: usize) -> crate::error::PmtResult { - let data = self.read(offset, length).await?; - - if data.len() == length { - Ok(data) - } else { - Err(UnexpectedNumberOfBytesReturned(length, data.len())) - } - } - - async fn read(&self, offset: usize, length: usize) -> crate::error::PmtResult { - let response = self - .bucket - .get_object_range( - self.pmtiles_path.as_str(), - offset as _, - Some((offset + length - 1) as _), - ) - .await?; - - let response_bytes = response.bytes(); - - if response_bytes.len() > length { - Err(ResponseBodyTooLong(response_bytes.len(), length)) - } else { - Ok(response_bytes.clone()) - } - } -} diff --git a/src/backend/http.rs b/src/backend_http.rs similarity index 58% rename from src/backend/http.rs rename to src/backend_http.rs index 917567e..56866ab 100644 --- a/src/backend/http.rs +++ b/src/backend_http.rs @@ -1,22 +1,47 @@ use async_trait::async_trait; use bytes::Bytes; -use reqwest::{ - header::{HeaderValue, RANGE}, - Client, IntoUrl, Method, Request, StatusCode, Url, -}; +use reqwest::header::{HeaderValue, RANGE}; +use reqwest::{Client, IntoUrl, Method, Request, StatusCode, Url}; -use crate::{async_reader::AsyncBackend, error::PmtResult, PmtError}; +use crate::async_reader::{AsyncBackend, AsyncPmTilesReader}; +use crate::cache::{DirectoryCache, NoCache}; +use crate::error::PmtResult; +use crate::PmtError; + +impl AsyncPmTilesReader { + /// Creates a new `PMTiles` reader from a URL using the Reqwest backend. + /// + /// Fails if [url] does not exist or is an invalid archive. (Note: HTTP requests are made to validate it.) + pub async fn new_with_url(client: Client, url: U) -> PmtResult { + Self::new_with_cached_url(NoCache, client, url).await + } +} + +impl AsyncPmTilesReader { + /// Creates a new `PMTiles` reader with cache from a URL using the Reqwest backend. + /// + /// Fails if [url] does not exist or is an invalid archive. (Note: HTTP requests are made to validate it.) + pub async fn new_with_cached_url( + cache: C, + client: Client, + url: U, + ) -> PmtResult { + let backend = HttpBackend::try_from(client, url)?; + + Self::try_from_cached_source(backend, cache).await + } +} pub struct HttpBackend { client: Client, - pmtiles_url: Url, + url: Url, } impl HttpBackend { pub fn try_from(client: Client, url: U) -> PmtResult { Ok(HttpBackend { client, - pmtiles_url: url.into_url()?, + url: url.into_url()?, }) } } @@ -41,7 +66,7 @@ impl AsyncBackend for HttpBackend { let range = format!("bytes={offset}-{end}"); let range = HeaderValue::try_from(range)?; - let mut req = Request::new(Method::GET, self.pmtiles_url.clone()); + let mut req = Request::new(Method::GET, self.url.clone()); req.headers_mut().insert(RANGE, range); let response = self.client.execute(req).await?.error_for_status()?; diff --git a/src/backend/mmap.rs b/src/backend_mmap.rs similarity index 60% rename from src/backend/mmap.rs rename to src/backend_mmap.rs index deb6230..deef24a 100644 --- a/src/backend/mmap.rs +++ b/src/backend_mmap.rs @@ -5,9 +5,30 @@ use async_trait::async_trait; use bytes::{Buf, Bytes}; use fmmap::tokio::{AsyncMmapFile, AsyncMmapFileExt as _, AsyncOptions}; -use crate::async_reader::AsyncBackend; +use crate::async_reader::{AsyncBackend, AsyncPmTilesReader}; +use crate::cache::{DirectoryCache, NoCache}; use crate::error::{PmtError, PmtResult}; +impl AsyncPmTilesReader { + /// Creates a new `PMTiles` reader from a file path using the async mmap backend. + /// + /// Fails if [p] does not exist or is an invalid archive. + pub async fn new_with_path>(path: P) -> PmtResult { + Self::new_with_cached_path(NoCache, path).await + } +} + +impl AsyncPmTilesReader { + /// Creates a new cached `PMTiles` reader from a file path using the async mmap backend. + /// + /// Fails if [p] does not exist or is an invalid archive. + pub async fn new_with_cached_path>(cache: C, path: P) -> PmtResult { + let backend = MmapBackend::try_from(path).await?; + + Self::try_from_cached_source(backend, cache).await + } +} + pub struct MmapBackend { file: AsyncMmapFile, } diff --git a/src/backend_s3.rs b/src/backend_s3.rs new file mode 100644 index 0000000..32fba59 --- /dev/null +++ b/src/backend_s3.rs @@ -0,0 +1,76 @@ +use async_trait::async_trait; +use bytes::Bytes; +use s3::Bucket; + +use crate::async_reader::{AsyncBackend, AsyncPmTilesReader}; +use crate::cache::{DirectoryCache, NoCache}; +use crate::error::PmtError::{ResponseBodyTooLong, UnexpectedNumberOfBytesReturned}; +use crate::PmtResult; + +impl AsyncPmTilesReader { + /// Creates a new `PMTiles` reader from a URL using the Reqwest backend. + /// + /// Fails if [url] does not exist or is an invalid archive. (Note: HTTP requests are made to validate it.) + pub async fn new_with_bucket_path(bucket: Bucket, path: String) -> PmtResult { + Self::new_with_cached_bucket_path(NoCache, bucket, path).await + } +} + +impl AsyncPmTilesReader { + /// Creates a new `PMTiles` reader with cache from a URL using the Reqwest backend. + /// + /// Fails if [url] does not exist or is an invalid archive. (Note: HTTP requests are made to validate it.) + pub async fn new_with_cached_bucket_path( + cache: C, + bucket: Bucket, + path: String, + ) -> PmtResult { + let backend = S3Backend::from(bucket, path); + + Self::try_from_cached_source(backend, cache).await + } +} + +pub struct S3Backend { + bucket: Bucket, + path: String, +} + +impl S3Backend { + #[must_use] + pub fn from(bucket: Bucket, path: String) -> S3Backend { + Self { bucket, path } + } +} + +#[async_trait] +impl AsyncBackend for S3Backend { + async fn read_exact(&self, offset: usize, length: usize) -> PmtResult { + let data = self.read(offset, length).await?; + + if data.len() == length { + Ok(data) + } else { + Err(UnexpectedNumberOfBytesReturned(length, data.len())) + } + } + + async fn read(&self, offset: usize, length: usize) -> PmtResult { + let response = self + .bucket + .get_object_range( + self.path.as_str(), + offset as _, + Some((offset + length - 1) as _), + ) + .await?; + + let response_bytes = response.bytes(); + + if response_bytes.len() > length { + Err(ResponseBodyTooLong(response_bytes.len(), length)) + } else { + Ok(response_bytes.clone()) + } + } +} diff --git a/src/error.rs b/src/error.rs index 3c022b6..0cfab96 100644 --- a/src/error.rs +++ b/src/error.rs @@ -29,21 +29,13 @@ pub enum PmtError { #[cfg(feature = "mmap-async-tokio")] #[error("Unable to open mmap file")] UnableToOpenMmapFile, - #[cfg(any( - feature = "http-async", - feature = "s3-async-native", - feature = "s3-async-rustls" - ))] + #[cfg(any(feature = "http-async", feature = "__async-s3"))] #[error("Unexpected number of bytes returned [expected: {0}, received: {1}].")] UnexpectedNumberOfBytesReturned(usize, usize), #[cfg(feature = "http-async")] #[error("Range requests unsupported")] RangeRequestsUnsupported, - #[cfg(any( - feature = "http-async", - feature = "s3-async-native", - feature = "s3-async-rustls" - ))] + #[cfg(any(feature = "http-async", feature = "__async-s3"))] #[error("HTTP response body is too long, Response {0}B > requested {1}B")] ResponseBodyTooLong(usize, usize), #[cfg(feature = "http-async")] @@ -52,7 +44,7 @@ pub enum PmtError { #[cfg(feature = "http-async")] #[error(transparent)] InvalidHeaderValue(#[from] reqwest::header::InvalidHeaderValue), - #[cfg(any(feature = "s3-async-rustls", feature = "s3-async-native"))] + #[cfg(feature = "__async-s3")] #[error(transparent)] S3(#[from] s3::error::S3Error), } diff --git a/src/header.rs b/src/header.rs index 89b6c94..f51aed3 100644 --- a/src/header.rs +++ b/src/header.rs @@ -5,7 +5,9 @@ use bytes::{Buf, Bytes}; use crate::error::{PmtError, PmtResult}; +#[cfg(feature = "__async")] pub(crate) const MAX_INITIAL_BYTES: usize = 16_384; +#[cfg(any(test, feature = "__async"))] pub(crate) const HEADER_SIZE: usize = 127; #[allow(dead_code)] diff --git a/src/lib.rs b/src/lib.rs index 31bebe9..a8fe6f3 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,50 +1,38 @@ #![forbid(unsafe_code)] -pub use directory::{DirEntry, Directory}; -pub use error::{PmtError, PmtResult}; - -pub use header::{Compression, Header, TileType}; - -#[cfg(any(feature = "s3-async-rustls", feature = "s3-async-native"))] -pub use backend::S3Backend; - -#[cfg(any(feature = "s3-async-rustls", feature = "s3-async-native"))] -pub use s3; - -#[cfg(feature = "http-async")] -pub use backend::HttpBackend; - +#[cfg(feature = "__async")] +pub mod async_reader; #[cfg(feature = "http-async")] -pub use reqwest; - +mod backend_http; #[cfg(feature = "mmap-async-tokio")] -pub use backend::MmapBackend; - -mod tile; - -mod header; - +mod backend_mmap; +#[cfg(feature = "__async-s3")] +mod backend_s3; +#[cfg(feature = "__async")] +pub mod cache; mod directory; - mod error; +mod header; +#[cfg(feature = "__async")] +mod tile; -mod backend; - -#[cfg(any( - feature = "http-async", - feature = "mmap-async-tokio", - feature = "s3-async-rustls", - feature = "s3-async-native" -))] -pub mod async_reader; - -#[cfg(any( - feature = "http-async", - feature = "mmap-async-tokio", - feature = "s3-async-native", - feature = "s3-async-rustls" -))] -pub mod cache; +#[cfg(feature = "http-async")] +pub use backend_http::HttpBackend; +#[cfg(feature = "mmap-async-tokio")] +pub use backend_mmap::MmapBackend; +#[cfg(feature = "__async-s3")] +pub use backend_s3::S3Backend; +pub use directory::{DirEntry, Directory}; +pub use error::{PmtError, PmtResult}; +pub use header::{Compression, Header, TileType}; +// +// Re-export crates exposed in our API to simplify dependency management +#[cfg(feature = "http-async")] +pub use reqwest; +#[cfg(feature = "__async-s3")] +pub use s3; +#[cfg(feature = "tilejson")] +pub use tilejson; #[cfg(test)] mod tests { diff --git a/src/tile.rs b/src/tile.rs index 1a3cf0a..f2fd86f 100644 --- a/src/tile.rs +++ b/src/tile.rs @@ -1,10 +1,42 @@ +#![allow(clippy::unreadable_literal)] + +const PYRAMID_SIZE_BY_ZOOM: [u64; 21] = [ + /* 0 */ 0, + /* 1 */ 1, + /* 2 */ 5, + /* 3 */ 21, + /* 4 */ 85, + /* 5 */ 341, + /* 6 */ 1365, + /* 7 */ 5461, + /* 8 */ 21845, + /* 9 */ 87381, + /* 10 */ 349525, + /* 11 */ 1398101, + /* 12 */ 5592405, + /* 13 */ 22369621, + /* 14 */ 89478485, + /* 15 */ 357913941, + /* 16 */ 1431655765, + /* 17 */ 5726623061, + /* 18 */ 22906492245, + /* 19 */ 91625968981, + /* 20 */ 366503875925, +]; + pub(crate) fn tile_id(z: u8, x: u64, y: u64) -> u64 { + // The 0/0/0 case is not needed for the base id computation, but it will fail hilbert_2d::u64::xy2h_discrete if z == 0 { return 0; } - // TODO: minor optimization with bit shifting - let base_id: u64 = 1 + (1..z).map(|i| 4u64.pow(u32::from(i))).sum::(); + let z_ind = usize::from(z); + let base_id = if z_ind < PYRAMID_SIZE_BY_ZOOM.len() { + PYRAMID_SIZE_BY_ZOOM[z_ind] + } else { + let last_ind = PYRAMID_SIZE_BY_ZOOM.len() - 1; + PYRAMID_SIZE_BY_ZOOM[last_ind] + (last_ind..z_ind).map(|i| 1_u64 << (i << 1)).sum::() + }; let tile_id = hilbert_2d::u64::xy2h_discrete(x, y, z.into(), hilbert_2d::Variant::Hilbert); @@ -21,5 +53,15 @@ mod test { assert_eq!(tile_id(1, 1, 0), 4); assert_eq!(tile_id(2, 1, 3), 11); assert_eq!(tile_id(3, 3, 0), 26); + assert_eq!(tile_id(20, 0, 0), 366503875925); + assert_eq!(tile_id(21, 0, 0), 1466015503701); + assert_eq!(tile_id(22, 0, 0), 5864062014805); + assert_eq!(tile_id(22, 0, 0), 5864062014805); + assert_eq!(tile_id(23, 0, 0), 23456248059221); + assert_eq!(tile_id(24, 0, 0), 93824992236885); + assert_eq!(tile_id(25, 0, 0), 375299968947541); + assert_eq!(tile_id(26, 0, 0), 1501199875790165); + assert_eq!(tile_id(27, 0, 0), 6004799503160661); + assert_eq!(tile_id(28, 0, 0), 24019198012642645); } }