Skip to content

Commit

Permalink
Fix granularity subtraction (#41)
Browse files Browse the repository at this point in the history
This PR fixes time subtraction when adjusting for OBJECT_STORE_DATA_GRANULARITY. By letting chrono subtract minutes from StorageSync time we can ensure right date and time for file URI

Fixes #40
  • Loading branch information
trueleo authored Aug 17, 2022
1 parent 97c6ad6 commit 4945757
Showing 1 changed file with 8 additions and 7 deletions.
15 changes: 8 additions & 7 deletions server/src/storage.rs
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ use crate::utils;

use async_trait::async_trait;
use bytes::Bytes;
use chrono::{Timelike, Utc};
use chrono::{Duration, Timelike, Utc};
use datafusion::arrow::record_batch::RecordBatch;
use serde::Serialize;

Expand Down Expand Up @@ -198,12 +198,13 @@ impl StorageSync {
let _storage_path = format!("{}/", CONFIG.storage.bucket_name());
let stream_name = self.path.replace(&local_path, "");
let parquet_path = format!("{}/data.parquet", self.path);
let uri = utils::date_to_prefix(self.time.date())
+ &utils::hour_to_prefix(self.time.hour())
// subtract OBJECT_STORE_DATA_GRANULARITY from current time here,
// this is because, when we're creating this file
// the data in the file is from OBJECT_STORE_DATA_GRANULARITY time ago.
+ &utils::minute_to_prefix(self.time.minute()-OBJECT_STORE_DATA_GRANULARITY, OBJECT_STORE_DATA_GRANULARITY).unwrap();
// subtract OBJECT_STORE_DATA_GRANULARITY from current time here,
// this is because, when we're creating this file
// the data in the file is from OBJECT_STORE_DATA_GRANULARITY time ago.
let time = self.time - Duration::minutes(OBJECT_STORE_DATA_GRANULARITY as i64);
let uri = utils::date_to_prefix(time.date())
+ &utils::hour_to_prefix(time.hour())
+ &utils::minute_to_prefix(time.minute(), OBJECT_STORE_DATA_GRANULARITY).unwrap();

let local_uri = str::replace(&uri, "/", ".");

Expand Down

0 comments on commit 4945757

Please sign in to comment.