Skip to content

Commit

Permalink
Add ability to create empty archive (#124)
Browse files Browse the repository at this point in the history
There was added new flag allow_empty to CreateArchiveJobBuilder. An archive can be created without any objects and it won't fail in that cases.
  • Loading branch information
Silverhorn27 authored Nov 17, 2023
1 parent a66aefa commit 3953715
Showing 1 changed file with 43 additions and 22 deletions.
65 changes: 43 additions & 22 deletions ssstar/src/create.rs
Original file line number Diff line number Diff line change
Expand Up @@ -175,32 +175,37 @@ impl CreateArchiveInput {
}
}

/// It's the same function like [`Self::into_possible_input_objects`] but with one exception
/// that if this object selector doesn't match any objects, an error is raised, since it likely
/// indicates a mistake on the user's part.
#[instrument(err, skip(self))]
async fn into_input_objects(self) -> Result<Vec<InputObject>> {
let input_text = self.to_string();
let input_objects = self.into_possible_input_objects().await?;
if input_objects.is_empty() {
crate::error::SelectorMatchesNoObjectsSnafu { input: input_text }.fail()
} else {
Ok(input_objects)
}
}

/// Evaluate the input against the actual object store API and return all objects that
/// corresond to this input.
///
/// This could be a long-running operation if a bucket or prefix is specified which contains
/// hundreds of thousands or millions of objects.
///
/// If this object selector doesn't match any objects, an error is raised, since it likely
/// indicates a mistake on the user's part.
#[instrument(err, skip(self))]
async fn into_input_objects(self) -> Result<Vec<InputObject>> {
async fn into_possible_input_objects(self) -> Result<Vec<InputObject>> {
// Enumerating objects is an object storage implementation-specific operation
let input_text = self.to_string();

debug!(self = %input_text, "Listing all object store objects that match this archive input");
let input_objects = self.bucket.list_matching_objects(self.selector).await?;
debug!(
self = %input_text,
count = input_objects.len(),
"Listing matching objects completed"
);

if input_objects.is_empty() {
crate::error::SelectorMatchesNoObjectsSnafu { input: input_text }.fail()
} else {
Ok(input_objects)
}
Ok(input_objects)
}
}

Expand Down Expand Up @@ -309,6 +314,7 @@ pub struct CreateArchiveJobBuilder {
config: Config,
target: TargetArchive,
inputs: Vec<CreateArchiveInput>,
allow_empty: bool,
}

impl CreateArchiveJobBuilder {
Expand All @@ -318,9 +324,16 @@ impl CreateArchiveJobBuilder {
config,
target,
inputs: vec![],
allow_empty: false,
}
}

/// Set a flag to indicate whether job can be created without objects.
/// By default it is `false`.
pub fn allow_empty(&mut self, allow: bool) {
self.allow_empty = allow;
}

/// Add one input URL to the job, validating the URL as part of the process.
///
/// Before adding this input, the URL will be parsed to extract the bucket name and object key,
Expand Down Expand Up @@ -364,16 +377,22 @@ impl CreateArchiveJobBuilder {
"Listing objects for all inputs"
);

let input_futs = self
.inputs
.into_iter()
.map(move |input| input.into_input_objects());

let mut inputs = futures::future::try_join_all(input_futs)
.await?
.into_iter()
.flatten()
.collect::<Vec<_>>();
let mut inputs = if self.allow_empty {
let input_futs = self
.inputs
.into_iter()
.map(move |input| input.into_possible_input_objects());
futures::future::try_join_all(input_futs).await?
} else {
let input_futs = self
.inputs
.into_iter()
.map(move |input| input.into_input_objects());
futures::future::try_join_all(input_futs).await?
}
.into_iter()
.flatten()
.collect::<Vec<_>>();

debug!(
object_count = inputs.len(),
Expand All @@ -399,6 +418,7 @@ impl CreateArchiveJobBuilder {
config: self.config,
target: self.target,
inputs,
allow_empty: self.allow_empty,
})
}
}
Expand Down Expand Up @@ -568,6 +588,7 @@ pub struct CreateArchiveJob {
config: Config,
target: TargetArchive,
inputs: Vec<InputObject>,
allow_empty: bool,
}

impl CreateArchiveJob {
Expand Down Expand Up @@ -609,7 +630,7 @@ impl CreateArchiveJob {
progress.input_objects_download_starting(total_objects, total_bytes);

// There must be at least one object otherwise it doesn't make sense to proceed
if total_objects == 0 {
if total_objects == 0 && !self.allow_empty {
return crate::error::NoInputsSnafu {}.fail();
}

Expand Down

0 comments on commit 3953715

Please sign in to comment.