Experimental chunk manifest support

CHANGELOG.md

@@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
  - Add a `makefile` and simplify `BUILD.md`
+ - Add experimental `chunk-manifest-json` storage transformer
 
 ### Changed
 - Bump `unsafe_cell_slice` to 0.2.0

zarrs/doc/status/storage_transformers.md

@@ -1 +1,3 @@
 Zarr V3 does not currently define any storage transformers.
+
+`zarrs` includes an experimental implementation of `chunk-manifest-json`.

zarrs/src/array/storage_transformer.rs

@@ -12,6 +12,8 @@ pub use storage_transformer_chain::StorageTransformerChain;
 mod storage_transformer_plugin;
 pub use storage_transformer_plugin::StorageTransformerPlugin;
 
+mod chunk_manifest_json; // NOTE: Intentionally private (but registered)
+
 use std::sync::Arc;
 
 use crate::{

zarrs/src/array/storage_transformer/chunk_manifest_json.rs

@@ -0,0 +1,61 @@
+use std::sync::Arc;
+
+use crate::metadata::v3::MetadataV3;
+
+use crate::node::NodePath;
+use crate::{
+    array::storage_transformer::StorageTransformerPlugin,
+    plugin::{PluginCreateError, PluginMetadataInvalidError},
+};
+
+use super::StorageTransformer;
+
+mod chunk_manifest_json_configuration;
+pub use chunk_manifest_json_configuration::ChunkManifestJsonConfiguration;
+
+mod chunk_manifest;
+pub use chunk_manifest::ChunkManifest;
+
+mod chunk_manifest_storage_transformer;
+pub use chunk_manifest_storage_transformer::ChunkManifestJsonStorageTransformer;
+
+pub const IDENTIFIER: &str = "chunk-manifest-json";
+
+// Register the storage plugin.
+inventory::submit! {
+    StorageTransformerPlugin::new(IDENTIFIER, is_name_chunk_manifest_json, create_storage_transformer_chunk_manifest_json)
+}
+
+fn is_name_chunk_manifest_json(name: &str) -> bool {
+    name.eq(IDENTIFIER)
+}
+
+pub(crate) fn create_storage_transformer_chunk_manifest_json(
+    metadata: &MetadataV3,
+    path: &NodePath,
+) -> Result<StorageTransformer, PluginCreateError> {
+    let configuration: ChunkManifestJsonConfiguration =
+        metadata.to_configuration().map_err(|_| {
+            PluginMetadataInvalidError::new(IDENTIFIER, "storage_transformer", metadata.clone())
+        })?;
+    let chunk_manifest_json: StorageTransformer = Arc::new(
+        ChunkManifestJsonStorageTransformer::new(configuration, path),
+    );
+    Ok(chunk_manifest_json)
+}
+
+#[cfg(test)]
+mod tests {
+    use std::{fs::File, io::BufReader};
+
+    use super::ChunkManifest;
+
+    #[test]
+    fn chunk_manifest() {
+        let file = File::open("tests/data/virtualizarr/virtualizarr.zarr/data/manifest.json")
+            .expect("Failed to open manifest file");
+        let reader = BufReader::new(file);
+        let chunk_manifest: Result<ChunkManifest, _> = serde_json::from_reader(reader);
+        assert!(chunk_manifest.is_ok());
+    }
+}

zarrs/src/array/storage_transformer/chunk_manifest_json/chunk_manifest.rs

@@ -0,0 +1,27 @@
+use std::{collections::HashMap, path::PathBuf};
+
+use derive_more::derive::Deref;
+use serde::{Deserialize, Serialize};
+
+#[derive(Serialize, Deserialize, Clone, Eq, PartialEq, Debug)]
+pub struct ChunkManifestValue {
+    pub path: PathBuf,
+    pub offset: u64,
+    pub length: u64,
+}
+
+#[derive(Clone, Eq, PartialEq, Debug, Deref)]
+pub struct ChunkManifest(HashMap<String, ChunkManifestValue>);
+
+impl<'de> serde::Deserialize<'de> for ChunkManifest {
+    fn deserialize<D: serde::Deserializer<'de>>(d: D) -> Result<Self, D::Error> {
+        let map = serde_json::Map::<String, serde_json::Value>::deserialize(d)?;
+        let mut map_out = HashMap::with_capacity(map.len());
+        for (key, v) in map {
+            let value: ChunkManifestValue = serde_json::from_value(v)
+                .map_err(|err| serde::de::Error::custom(err.to_string()))?;
+            map_out.insert(key, value);
+        }
+        Ok(ChunkManifest(map_out))
+    }
+}

zarrs/src/array/storage_transformer/chunk_manifest_json/chunk_manifest_json_configuration.rs

@@ -0,0 +1,10 @@
+use std::path::PathBuf;
+
+use serde::{Deserialize, Serialize};
+
+#[derive(Serialize, Deserialize, Clone, Eq, PartialEq, Debug)]
+#[serde(deny_unknown_fields)]
+pub struct ChunkManifestJsonConfiguration {
+    /// The manifest path.
+    pub manifest: PathBuf,
+}