diff --git a/.github/actions/spelling/allow.txt b/.github/actions/spelling/allow.txt index 690b830ba2fec..ef56425a13385 100644 --- a/.github/actions/spelling/allow.txt +++ b/.github/actions/spelling/allow.txt @@ -113,6 +113,7 @@ Mediacom Medion Meizu Mertz +metakey Metakey Micromax Mito diff --git a/Cargo.lock b/Cargo.lock index ee814e002ddf8..62ef53be828b2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1334,7 +1334,7 @@ dependencies = [ "log", "paste", "pin-project", - "quick-xml 0.30.0", + "quick-xml", "rand 0.8.5", "reqwest", "rustc_version 0.4.0", @@ -5830,9 +5830,9 @@ checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" [[package]] name = "opendal" -version = "0.38.1" +version = "0.41.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4df645b6012162c04c8949e9b96ae2ef002e79189cfb154e507e51ac5be76a09" +checksum = "e31b48f0af6de5b3b344c1acc1e06c4581dca3e13cd5ba05269927fc2abf953a" dependencies = [ "anyhow", "async-compat", @@ -5851,7 +5851,7 @@ dependencies = [ "parking_lot", "percent-encoding", "pin-project", - "quick-xml 0.27.1", + "quick-xml", "reqwest", "serde", "serde_json", @@ -6941,16 +6941,6 @@ version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" -[[package]] -name = "quick-xml" -version = "0.27.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffc053f057dd768a56f62cd7e434c42c831d296968997e9ac1f76ea7c2d14c41" -dependencies = [ - "memchr", - "serde", -] - [[package]] name = "quick-xml" version = "0.30.0" diff --git a/Cargo.toml b/Cargo.toml index 968a5f5162170..ea84d07e1c225 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -184,7 +184,7 @@ azure_storage = { version = "0.16", default-features = false, optional = true } azure_storage_blobs = { version = "0.16", default-features = false, optional = true } # OpenDAL -opendal = {version = "0.38", default-features = false, features = ["native-tls", "services-webhdfs"], optional = true} +opendal = {version = "0.41", default-features = false, features = ["native-tls", "services-webhdfs"], optional = true} # Tower tower = { version = "0.4.13", default-features = false, features = ["buffer", "limit", "retry", "timeout", "util", "balance", "discover"] } diff --git a/src/sinks/webhdfs/integration_tests.rs b/src/sinks/webhdfs/integration_tests.rs index 0388da9246af9..d57101e3473b4 100644 --- a/src/sinks/webhdfs/integration_tests.rs +++ b/src/sinks/webhdfs/integration_tests.rs @@ -3,7 +3,7 @@ use std::{ time::Duration, }; -use futures::{stream, StreamExt}; +use futures::stream; use opendal::{Entry, Metakey}; use similar_asserts::assert_eq; use vector_lib::codecs::{encoding::FramingConfig, TextSerializerConfig}; @@ -74,25 +74,12 @@ async fn hdfs_rotate_files_after_the_buffer_size_is_reached() { // Hard-coded sleeps are bad, but we're waiting on localstack's state to converge. tokio::time::sleep(Duration::from_secs(1)).await; - // blocking_scan isn't supported - let objects: Vec = op - .scan("/") + let mut objects: Vec = op + .list_with("/") + .delimiter("") + .metakey(Metakey::Mode) .await - .unwrap() - .map(|x| x.unwrap()) - .collect() - .await; - - let mut objects = objects - .into_iter() - .filter(|entry| { - op.blocking() - .metadata(entry, Metakey::Mode) - .unwrap() - .mode() - .is_file() - }) - .collect::>(); + .unwrap(); // Sort file path in order, because we have the event id in path. objects.sort_by(|l, r| l.path().cmp(r.path()));