Merge remote-tracking branch 'origin/feature/nuxt_frontend'

server/chroma/Cargo.toml

@@ -30,6 +30,7 @@ time = "0.3.28"
 kamadak-exif = "0.5.5"
 cabbage = "0.1.2"
 moka = { version = "0.12.5", features = ["future"] }
+dotenv = "0.15.0"
 
 [dev-dependencies]
 serde_json = "1.0.93"

server/chroma/src/config.rs

@@ -18,10 +18,6 @@ pub struct Config {
     /// Database connection url
     pub db_url: Option<String>,
 
-    /// The storage engine to be used
-    #[serde(default = "default_storage_engine")]
-    pub storage_engine: StorageEngine,
-
     /// The name of the S3 bucket that should be used
     /// Required if `storage_engine` is set to [StorageEngine::S3].
     pub s3_bucket_name: Option<String>,
@@ -43,10 +39,6 @@ pub struct Config {
     /// Amazon S3.
     pub s3_force_path_style: Option<bool>,
 
-    /// The base path for storage.
-    /// Required if `storage_engine` is set to [StorageEngine::File].
-    pub file_base: Option<String>,
-
     /// OAuth2 client ID created in Koala
     pub koala_client_id: String,
     /// OAuth2 client secret created in Koala.
@@ -75,7 +67,7 @@ pub struct Config {
     /// If not provided, the default [Config::DEFAULT_KOALA_USER_AGENT] will be used.
     koala_user_agent: Option<String>,
     /// The URI to which Chroma should redirect after the user
-    /// has logged in. Refer to the UI's documentation
+    /// has logged in. Refer to the UIs documentation
     /// for what this value should be.
     /// The query parameters `session_id` and `is_admin` will be appended.
     /// No existing query parameters should be in the URI.
@@ -95,16 +87,6 @@ pub struct Config {
     // ANCHOR_END: config
 }
 
-const fn default_storage_engine() -> StorageEngine {
-    StorageEngine::S3
-}
-
-#[derive(Debug, Clone, Deserialize)]
-pub enum StorageEngine {
-    S3,
-    File,
-}
-
 impl Config {
     /// The default user agent for Koala when none is configured
     const DEFAULT_KOALA_USER_AGENT: &'static str = "Chroma server";
@@ -185,35 +167,24 @@ impl Config {
     /// Check if the configuration is valid.
     /// Returns `true` if it is, `false` if it is not.
     pub fn validate(&self) -> bool {
-        // Validate storage engine options
-        match self.storage_engine {
-            StorageEngine::File => {
-                if self.file_base.is_none() {
-                    warn!("Config validation failed on File fields");
-                    return false;
-                }
-            }
-            StorageEngine::S3 => {
-                let check_field = |fname: &'static str, f: &Option<_>| {
-                    if f.is_none() {
-                        warn!("Config validation failed on S3_{fname}");
-                        false
-                    } else {
-                        true
-                    }
-                };
-
-                let config_ok = check_field("ACCESS_KEY_ID", &self.s3_access_key_id)
-                    && check_field("SECRET_ACCESS_KEY", &self.s3_secret_access_key)
-                    && check_field("BUCKET_NAME", &self.s3_bucket_name)
-                    && check_field("ENDPOINT_URL", &self.s3_endpoint_url)
-                    && check_field("REGION", &self.s3_region);
-
-                if !config_ok {
-                    warn!("Config validation failed.");
-                    return false;
-                }
+        let check_field = |field_name: &'static str, field_value: &Option<_>| {
+            if field_value.is_none() {
+                warn!("Config validation failed on S3_{field_name}");
+                false
+            } else {
+                true
             }
+        };
+
+        let config_ok = check_field("ACCESS_KEY_ID", &self.s3_access_key_id)
+            && check_field("SECRET_ACCESS_KEY", &self.s3_secret_access_key)
+            && check_field("BUCKET_NAME", &self.s3_bucket_name)
+            && check_field("ENDPOINT_URL", &self.s3_endpoint_url)
+            && check_field("REGION", &self.s3_region);
+
+        if !config_ok {
+            warn!("Config validation failed.");
+            return false;
         }
 
         true

server/chroma/src/main.rs

@@ -9,10 +9,9 @@ use cabbage::KoalaApi;
 use color_eyre::eyre::Error;
 use color_eyre::Result;
 use dal::database::Database;
-use dal::s3::S3Config;
-use dal::storage_engine::StorageEngine;
+use dal::storage_engine::{S3Config, Storage};
+use dotenv::dotenv;
 use noiseless_tracing_actix_web::NoiselessRootSpanBuilder;
-use std::path::PathBuf;
 use std::time::Duration;
 use tracing::{info, warn};
 use tracing_actix_web::TracingLogger;
@@ -26,7 +25,8 @@ mod routes;
 
 #[tokio::main]
 async fn main() -> Result<()> {
-    color_eyre::install()?;
+    dotenv().ok();
+
     install_tracing();
 
     info!("Starting");
@@ -45,23 +45,16 @@ async fn main() -> Result<()> {
     info!("Initializing database");
     let db = Database::new(config.database_config()?).await?;
 
-    info!("Initializing storage engine");
-    let storage = match config.storage_engine {
-        config::StorageEngine::S3 => {
-            StorageEngine::new_s3(S3Config {
-                bucket_name: config.s3_bucket_name.clone().unwrap(),
-                endpoint_url: config.s3_endpoint_url.clone().unwrap(),
-                region: config.s3_region.clone().unwrap(),
-                access_key_id: config.s3_access_key_id.clone().unwrap(),
-                secret_access_key: config.s3_secret_access_key.clone().unwrap(),
-                use_path_style: config.s3_force_path_style(),
-            })
-            .await?
-        }
-        config::StorageEngine::File => {
-            StorageEngine::new_file(PathBuf::from(config.file_base.clone().unwrap())).await?
-        }
-    };
+    info!("Initializing S3 storage engine");
+    let storage = Storage::new(S3Config {
+        bucket_name: config.s3_bucket_name.clone().unwrap(),
+        endpoint_url: config.s3_endpoint_url.clone().unwrap(),
+        region: config.s3_region.clone().unwrap(),
+        access_key_id: config.s3_access_key_id.clone().unwrap(),
+        secret_access_key: config.s3_secret_access_key.clone().unwrap(),
+        use_path_style: config.s3_force_path_style(),
+    })
+    .await?;
 
     let appdata = AppData {
         koala: KoalaApi::new(config.koala_base_redirect_uri().clone())?,

server/chroma/src/routes/appdata.rs

@@ -3,7 +3,7 @@ use crate::routes::authorization::Authorization;
 use actix_web::web;
 use cabbage::KoalaApi;
 use dal::database::{Album, Database};
-use dal::storage_engine::StorageEngine;
+use dal::storage_engine::Storage;
 use moka::future::Cache;
 
 pub type WebData = web::Data<AppData>;
@@ -13,7 +13,7 @@ pub type AlbumIdCache = Cache<String, Album>;
 #[derive(Debug, Clone)]
 pub struct AppData {
     pub db: Database,
-    pub storage: StorageEngine,
+    pub storage: Storage,
     pub config: Config,
     pub koala: KoalaApi,
 }

server/chroma/src/routes/error.rs

@@ -13,7 +13,7 @@ pub enum Error {
     #[error("Bad request: {0}")]
     BadRequest(String),
     #[error("Internal server error")]
-    StorageEngine(#[from] dal::storage_engine::StorageEngineError),
+    StorageEngine(#[from] dal::storage_engine::error::StorageError),
     #[error("Something went wrong on Koala's end")]
     Koala(reqwest::Error),
     #[error("The requested resource may not be accessed by the authorized user.")]

server/chroma/src/routes/v1/album/get.rs

@@ -4,7 +4,7 @@ use crate::routes::error::{Error, WebResult};
 use actix_multiresponse::Payload;
 use actix_web::web;
 use dal::database::{Album, Photo};
-use dal::storage_engine::{EngineType, PhotoQuality};
+use dal::storage_engine::PhotoQuality;
 use dal::DalError;
 use futures::future::join_all;
 use proto::{AlbumWithCoverPhoto, GetAlbumResponse};
@@ -67,22 +67,13 @@ pub async fn get(
                 .await?
                 .ok_or(Error::NotFound)?;
 
-            let photo = match data.storage.engine_type() {
-                EngineType::S3 => {
-                    photo
-                        .photo_to_proto_url(&data.storage, PhotoQuality::W400)
-                        .await
-                }
-                EngineType::File => {
-                    photo
-                        .photo_to_proto_bytes(&data.storage, PhotoQuality::W400)
-                        .await
-                }
-            }
-            .map_err(|e| match e {
-                DalError::Storage(e) => Error::from(e),
-                DalError::Db(e) => Error::from(e),
-            })?;
+            let photo = photo
+                .photo_to_proto_url(&data.storage, PhotoQuality::W400)
+                .await
+                .map_err(|e| match e {
+                    DalError::Storage(e) => Error::from(e),
+                    DalError::Db(e) => Error::from(e),
+                })?;
 
             Some(photo)
         } else {

server/chroma/src/routes/v1/album/list.rs

@@ -5,9 +5,8 @@ use crate::routes::v1::PhotoQuality;
 use actix_multiresponse::Payload;
 use actix_web::web;
 use dal::database::{Album, Photo};
-use dal::s3::aws_errors::GetObjectErrorKind;
-use dal::s3::{S3Error, SdkError};
-use dal::storage_engine::{EngineType, StorageEngineError};
+use dal::storage_engine::aws_error::GetObjectErrorKind;
+use dal::storage_engine::error::{SdkError, StorageError};
 use dal::DalError;
 use futures::future::{join_all, try_join_all};
 use proto::{AlbumWithCoverPhoto, ListAlbumsResponse};
@@ -105,42 +104,26 @@ pub async fn list(
                                 qpref
                             };
 
-                            let photo = match storage.engine_type() {
-                                EngineType::S3 => match photo
-                                    .photo_to_proto_url(&storage, quality)
-                                    .await
-                                {
-                                    Ok(v) => v,
-                                    Err(e) => {
-                                        return match &e {
-                                            DalError::Storage(s) => match s {
-                                                StorageEngineError::S3(s) => match s {
-                                                    S3Error::GetObject(s) => match s {
-                                                        SdkError::ServiceError(s) => match s
-                                                            .err()
-                                                            .kind
-                                                        {
-                                                            GetObjectErrorKind::NoSuchKey(_) => {
-                                                                album_id_cache
-                                                                    .remove(&album.id)
-                                                                    .await;
-                                                                album.delete(&database).await?;
-                                                                Ok(None)
-                                                            }
-                                                            _ => Err(e),
-                                                        },
-                                                        _ => Err(e),
-                                                    },
+                            let photo = match photo.photo_to_proto_url(&storage, quality).await {
+                                Ok(v) => v,
+                                Err(e) => {
+                                    return match &e {
+                                        DalError::Storage(s) => match s {
+                                            StorageError::GetObject(s) => match s {
+                                                SdkError::ServiceError(s) => match s.err().kind {
+                                                    GetObjectErrorKind::NoSuchKey(_) => {
+                                                        album_id_cache.remove(&album.id).await;
+                                                        album.delete(&database).await?;
+                                                        Ok(None)
+                                                    }
                                                     _ => Err(e),
                                                 },
                                                 _ => Err(e),
                                             },
                                             _ => Err(e),
-                                        }
+                                        },
+                                        _ => Err(e),
                                     }
-                                },
-                                EngineType::File => {
-                                    photo.photo_to_proto_bytes(&storage, quality).await?
                                 }
                             };
 

server/chroma/src/routes/v1/photo/create.rs

@@ -3,7 +3,7 @@ use crate::routes::authorization::Authorization;
 use crate::routes::error::{Error, ImagePipelineError, WebResult};
 use actix_multiresponse::Payload;
 use dal::database::{Album, Database, Photo};
-use dal::storage_engine::{PhotoQuality, StorageEngine};
+use dal::storage_engine::{PhotoQuality, Storage};
 use exif::{In, Tag};
 use image::imageops::FilterType;
 use image::io::Reader;
@@ -120,7 +120,7 @@ async fn image_pipeline(
             PhotoQuality::Original
         );
         match engine
-            .create_photo(&photo_id, image, PhotoQuality::Original)
+            .create_photo(&photo_id, PhotoQuality::Original, image)
             .await
         {
             Ok(_) => {}
@@ -157,7 +157,7 @@ async fn image_pipeline(
 fn resize_and_save(
     image: DynamicImage,
     quality: PhotoQuality,
-    engine: StorageEngine,
+    engine: Storage,
     db: Database,
     photo_id: String,
 ) {
@@ -181,7 +181,7 @@ fn resize_and_save(
 
         trace!("Saving image '{photo_id}' in quality '{quality:?}'");
         match engine
-            .create_photo(&photo_id, converted_image_data, quality.clone())
+            .create_photo(&photo_id, quality.clone(), converted_image_data)
             .await
         {
             Ok(_) => {}

server/chroma/src/routes/v1/photo/get.rs

@@ -5,7 +5,6 @@ use crate::routes::v1::PhotoQuality;
 use actix_multiresponse::Payload;
 use actix_web::web;
 use dal::database::Photo;
-use dal::storage_engine::StorageEngineError;
 use dal::DalError;
 use image::{DynamicImage, ImageOutputFormat};
 use proto::photo_respone::Response;
@@ -58,23 +57,19 @@ pub async fn get(
         .ok_or(Error::NotFound)?;
 
     if query.format.eq(&ImageFormat::WebP) && !query.force_bytes {
-        match photo
+        return match photo
             .clone()
             .photo_to_proto_url(&data.storage, query.quality_preference.clone().into())
             .await
         {
-            Ok(p) => {
-                return Ok(Payload(GetPhotoResponse { photo: Some(p) }));
-            }
+            Ok(p) => Ok(Payload(GetPhotoResponse { photo: Some(p) })),
             Err(e) => match e {
                 DalError::Storage(e) => match e {
-                    // URL mode is not supported
-                    StorageEngineError::NotSupported => {}
-                    _ => return Err(e.into()),
+                    _ => Err(e.into()),
                 },
-                DalError::Db(e) => return Err(e.into()),
+                DalError::Db(e) => Err(e.into()),
             },
-        }
+        };
     }
 
     let mut proto = photo

server/chroma/src/routes/v1/photo/list.rs

@@ -5,7 +5,6 @@ use crate::routes::v1::PhotoQuality;
 use actix_multiresponse::Payload;
 use actix_web::web;
 use dal::database::Photo;
-use dal::storage_engine::EngineType;
 use dal::DalError;
 use futures::future::join_all;
 use proto::ListPhotoResponse;
@@ -43,13 +42,7 @@ pub async fn list(
         let storage = data.storage.clone();
         let qpref: dal::storage_engine::PhotoQuality = query.quality_preference.clone().into();
 
-        async move {
-            match storage.engine_type() {
-                EngineType::S3 => p.photo_to_proto_url(&storage, qpref).await,
-                // File engine doesn't support URL mode
-                EngineType::File => p.photo_to_proto_bytes(&storage, qpref).await,
-            }
-        }
+        async move { p.photo_to_proto_url(&storage, qpref).await }
     }))
     .await
     .into_iter()

server/dal/migrations/6_typo_photo_metadata_index_fix.sql

@@ -0,0 +1 @@
+ALTER INDEX idx_photo_metaddata_album_id RENAME TO idx_photo_metadata_album_id