File uploads login page (#7)

* feat(backend and login): started file upload logic [2024-10-25] * refactor(UI): small refactors to comments and variables [2024-10-25] * feat: add login page [2024-10-26] * feat: handle login http call [2024-10-26] * style(Login Page): Seperated out login page logic and added in 3 grid to page background [2024-10-26] * fix!(Backend): Attempts to fix Backend and make upload_layer endpoint work - 2 type errros left to fix [2024-10-26] BREAKING CHANGE: Attempts to fix Backend and make upload_layer endpoint work - 2 type errros left to fix * fix!(rust backend): Further attempts to fix rust backend for upload layer [2024-10-27] BREAKING CHANGE: Further attempts to fix rust backend for upload layer * fix!(rust backend): Another fix on backend - fixing references and ownership when creating / uploading a layer to Dynamo and Postgis [2024-10-27] BREAKING CHANGE: Another fix on backend - fixing references and ownership when creating / uploading a layer to Dynamo and Postgis * fix: update upload layer payload [2024-10-27] --------- Co-authored-by: Chris Carlon <[email protected]>
enmeshed-analytics · Oct 27, 2024 · 7e7dbf2 · 7e7dbf2
1 parent 8799970
commit 7e7dbf2
Show file tree

Hide file tree

Showing 28 changed files with 1,076 additions and 339 deletions.
diff --git a/gridwalk-backend/Cargo.toml b/gridwalk-backend/Cargo.toml
@@ -13,7 +13,7 @@ axum = { version = "0.7", features = ["macros", "multipart"] }
 axum-extra = { version = "0.9.4", features = ["typed-header"] }
 base64 = "0.22.1"
 deadpool-postgres = "0.14.0"
-duckdb-postgis = "0.1.4"
+duckdb-postgis = "0.1.5"
 geozero = { version = "0.14.0", features = [
     "with-postgis-postgres",
     "with-postgis-sqlx",

diff --git a/gridwalk-backend/src/core/layer.rs b/gridwalk-backend/src/core/layer.rs
@@ -1,13 +1,19 @@
+use super::{get_unix_timestamp, Workspace, WorkspaceRole};
 use crate::core::User;
 use crate::data::Database;
 use anyhow::{anyhow, Result};
 use duckdb_postgis;
 use serde::{Deserialize, Serialize};
+use std::sync::Arc;
 use uuid::Uuid;
 
-use super::{get_unix_timestamp, Workspace, WorkspaceRole};
+#[derive(Debug, Clone, Deserialize, Serialize)]
+pub struct CreateLayer {
+    pub name: String,
+    pub workspace_id: String,
+}
 
-#[derive(Debug, Clone, Serialize)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct Layer {
     pub id: String,
     pub workspace_id: String,
@@ -16,57 +22,49 @@ pub struct Layer {
     pub created_at: u64,
 }
 
-#[derive(Debug, Clone, Deserialize, Serialize)]
-pub struct CreateLayer {
-    pub name: String,
-    pub workspace_id: String,
-}
-
 impl Layer {
-    pub fn from_req(req: CreateLayer, user: User) -> Self {
+    pub fn from_req(req: CreateLayer, user: &User) -> Self {
         Layer {
             id: Uuid::new_v4().to_string(),
             workspace_id: req.workspace_id,
             name: req.name,
-            uploaded_by: user.id,
+            uploaded_by: user.id.clone(),
             created_at: get_unix_timestamp(),
         }
     }
-}
 
-impl Layer {
-    pub async fn create<D: Database>(
-        self,
-        database: D,
+    pub async fn create(
+        &self,
+        database: &Arc<dyn Database>,
         user: &User,
-        wsp: &Workspace,
-        layer: &Layer,
+        workspace: &Workspace,
     ) -> Result<()> {
         // Get workspace member record
-        let requesting_member = wsp
-            .clone()
-            .get_member(database.clone(), user.clone())
-            .await?;
-
+        let requesting_member = workspace.get_member(database, user).await?;
         if requesting_member.role == WorkspaceRole::Read {
-            Err(anyhow!("User does not have permissions to create layers."))?
+            return Err(anyhow!("User does not have permissions to create layers."));
         }
-        database.create_layer(layer).await?;
-
+        database.create_layer(self).await?;
         Ok(())
     }
 
-    pub async fn send_to_postgis(self, file_path: &str) -> Result<()> {
-        let postgis_uri = "postgresql://admin:password@localhost:5432/gridwalk";
-        let schema =
-            duckdb_postgis::duckdb_load::launch_process_file(file_path, &self.id, postgis_uri)?;
-
-        println!("{schema:?}");
+    pub async fn send_to_postgis(&self, file_path: &str) -> Result<()> {
+        let postgis_uri = std::env::var("POSTGIS_URI")
+            .map_err(|_| anyhow!("PostGIS URI not set in environment variables"))?;
+        let schema = duckdb_postgis::duckdb_load::launch_process_file(
+            file_path,
+            &self.id,
+            &postgis_uri,
+            &self.workspace_id,
+        )
+        .map_err(|e| anyhow!("Failed to send file to PostGIS: {:?}", e))?;
+        println!("{:?}", schema);
         Ok(())
     }
 
-    pub async fn write_record<D: Database>(self, database: D) -> Result<()> {
-        database.create_layer(&self).await?;
+    // Change this to match the pattern used elsewhere
+    pub async fn write_record(&self, database: &Arc<dyn Database>) -> Result<()> {
+        database.create_layer(self).await?;
         Ok(())
     }
 }
diff --git a/gridwalk-backend/src/core/mod.rs b/gridwalk-backend/src/core/mod.rs
@@ -1,11 +1,13 @@
 pub mod common;
 pub mod connector;
+pub mod layer;
 pub mod session;
 pub mod user;
 pub mod workspace;
 
 pub use common::*;
 pub use connector::*;
+pub use layer::*;
 pub use session::*;
 pub use user::*;
 pub use workspace::*;
diff --git a/gridwalk-backend/src/core/workspace.rs b/gridwalk-backend/src/core/workspace.rs
@@ -86,7 +86,7 @@ impl Workspace {
         user: &User,
         role: WorkspaceRole,
     ) -> Result<()> {
-        let requesting_member = self.clone().get_member(&database, req_user.clone()).await?;
+        let requesting_member = self.clone().get_member(&database, &req_user).await?;
 
         println!(
             "{} is {} of the {} workspace",
@@ -105,12 +105,12 @@ impl Workspace {
     }
 
     pub async fn get_member(
-        self,
+        &self,
         database: &Arc<dyn Database>,
-        user: User,
+        user: &User,
     ) -> Result<WorkspaceMember> {
         // TODO: Fix unwrap
-        Ok(database.get_workspace_member(self, user).await.unwrap())
+        Ok(database.get_workspace_member(&self, user).await.unwrap())
     }
 
     pub async fn remove_member(
@@ -119,7 +119,7 @@ impl Workspace {
         req_user: &User,
         user: &User,
     ) -> Result<()> {
-        let requesting_member = self.clone().get_member(&database, req_user.clone()).await?;
+        let requesting_member = self.clone().get_member(&database, &req_user).await?;
 
         println!(
             "{} is {} of the {} workspace",

diff --git a/gridwalk-backend/src/data/config.rs b/gridwalk-backend/src/data/config.rs
@@ -1,4 +1,4 @@
-use crate::core::{Connection, Session, User, Workspace, WorkspaceMember, WorkspaceRole};
+use crate::core::{Connection, Layer, Session, User, Workspace, WorkspaceMember, WorkspaceRole};
 use anyhow::Result;
 use async_trait::async_trait;
 
@@ -20,7 +20,7 @@ pub trait UserStore: Send + Sync + 'static {
         role: WorkspaceRole,
         joined_at: u64,
     ) -> Result<()>;
-    async fn get_workspace_member(&self, wsp: Workspace, user: User) -> Result<WorkspaceMember>;
+    async fn get_workspace_member(&self, wsp: &Workspace, user: &User) -> Result<WorkspaceMember>;
     async fn remove_workspace_member(&self, org: &Workspace, user: &User) -> Result<()>;
     async fn create_connection(&self, connection: &Connection) -> Result<()>;
     async fn get_workspace_connection(
@@ -29,6 +29,7 @@ pub trait UserStore: Send + Sync + 'static {
         connection_id: &str,
     ) -> Result<Connection>;
     async fn get_workspace_connections(&self, workspace_id: &str) -> Result<Vec<Connection>>;
+    async fn create_layer(&self, layer: &Layer) -> Result<()>;
 }
 
 #[async_trait]

diff --git a/gridwalk-backend/src/data/dynamodb/config.rs b/gridwalk-backend/src/data/dynamodb/config.rs
@@ -1,4 +1,6 @@
-use crate::core::{Connection, CreateUser, Email, User, Workspace, WorkspaceMember, WorkspaceRole};
+use crate::core::{
+    Connection, CreateUser, Email, Layer, User, Workspace, WorkspaceMember, WorkspaceRole,
+};
 use crate::data::{Database, UserStore};
 use anyhow::{anyhow, Result};
 use async_trait::async_trait;
@@ -304,7 +306,7 @@ impl UserStore for Dynamodb {
         Ok(())
     }
 
-    async fn get_workspace_member(&self, wsp: Workspace, user: User) -> Result<WorkspaceMember> {
+    async fn get_workspace_member(&self, wsp: &Workspace, user: &User) -> Result<WorkspaceMember> {
         println!("{wsp:?}");
         let pk = format!("WSP#{0}", wsp.id);
         let sk = format!("USER#{0}", user.id);
@@ -394,7 +396,6 @@ impl UserStore for Dynamodb {
             Err(_e) => Err(anyhow!("workspace not found")),
         }
     }
-
     async fn get_workspace_connections(&self, workspace_id: &str) -> Result<Vec<Connection>> {
         let pk = format!("WSP#{}", workspace_id);
 
@@ -417,4 +418,33 @@ impl UserStore for Dynamodb {
             None => Ok(vec![]),
         }
     }
+
+    async fn create_layer(&self, layer: &Layer) -> Result<()> {
+        // Create the workspace member item to insert
+        let mut item = std::collections::HashMap::new();
+
+        item.insert(
+            String::from("PK"),
+            AV::S(format!("WSP#{}", layer.workspace_id)),
+        );
+        item.insert(String::from("SK"), AV::S(format!("LAYER#{}", layer.id)));
+        item.insert(String::from("name"), AV::S(layer.clone().name));
+        item.insert(
+            String::from("uploaded_by"),
+            AV::S(layer.clone().uploaded_by),
+        );
+        item.insert(
+            String::from("created_at"),
+            AV::N(layer.created_at.to_string()),
+        );
+
+        self.client
+            .put_item()
+            .table_name(&self.table_name)
+            .set_item(Some(item))
+            .send()
+            .await?;
+
+        Ok(())
+    }
 }
diff --git a/gridwalk-backend/src/main.rs b/gridwalk-backend/src/main.rs
@@ -23,6 +23,7 @@ async fn main() -> Result<()> {
     // Create GeospatialConfig
     let geospatial_config = GeospatialConfig::new();
 
+    // Create initial App State
     let app_state = AppState {
         app_data: app_db,
         geospatial_config,

diff --git a/gridwalk-backend/src/routes/connector.rs b/gridwalk-backend/src/routes/connector.rs
@@ -68,7 +68,7 @@ pub async fn list_sources(
         .await
         .unwrap();
     let _member = workspace
-        .get_member(&state.app_data, auth_user.user.unwrap())
+        .get_member(&state.app_data, &auth_user.user.unwrap())
         .await
         .unwrap();
     // Check member role

diff --git a/gridwalk-backend/src/routes/layer.rs b/gridwalk-backend/src/routes/layer.rs
@@ -1,7 +1,6 @@
+use crate::app_state::AppState;
 use crate::auth::AuthUser;
-use crate::core::{Layer, Workspace, WorkspaceRole};
-use crate::data::Database;
-use crate::{app_state::AppState, core::CreateLayer};
+use crate::core::{CreateLayer, Layer, Workspace, WorkspaceRole};
 use axum::{
     extract::{Extension, Multipart, State},
     http::StatusCode,
@@ -16,14 +15,18 @@ use tokio::{
     io::AsyncWriteExt,
 };
 
-pub async fn upload_layer<D: Database>(
-    State(state): State<Arc<AppState<D>>>,
+// Remove the generic type parameter and use Arc<AppState> directly
+pub async fn upload_layer(
+    State(state): State<Arc<AppState>>,
     Extension(auth_user): Extension<AuthUser>,
     mut multipart: Multipart,
 ) -> Result<impl IntoResponse, StatusCode> {
+    // Early return if no user
+    let user = auth_user.user.as_ref().ok_or(StatusCode::UNAUTHORIZED)?;
     let mut file_data = Vec::new();
     let mut layer_info: Option<CreateLayer> = None;
 
+    // Process multipart form data
     while let Some(field) = multipart
         .next_field()
         .await
@@ -32,7 +35,6 @@ pub async fn upload_layer<D: Database>(
         let name = field.name().ok_or(StatusCode::BAD_REQUEST)?.to_string();
         match name.as_str() {
             "file" => {
-                println!("FILE");
                 file_data = field
                     .bytes()
                     .await
@@ -57,25 +59,25 @@ pub async fn upload_layer<D: Database>(
     }
 
     let layer_info = layer_info.ok_or(StatusCode::BAD_REQUEST)?;
-    let layer = Layer::from_req(layer_info, auth_user.user.clone().unwrap());
-    let workspace = Workspace::from_id(state.app_data.clone(), &layer.workspace_id)
+    let layer = Layer::from_req(layer_info, user);
+
+    // Get workspace and check permissions
+    let workspace = Workspace::from_id(&state.app_data, &layer.workspace_id)
         .await
-        .unwrap();
+        .map_err(|_| StatusCode::NOT_FOUND)?;
+
     let member = workspace
-        .get_member(state.app_data.clone(), auth_user.user.unwrap())
+        .get_member(&state.app_data, &user)
         .await
-        .unwrap();
+        .map_err(|_| StatusCode::FORBIDDEN)?;
 
-    // Check requesting user workspace permissions
     if member.role == WorkspaceRole::Read {
-        return Ok((StatusCode::FORBIDDEN, format!("")));
+        return Ok((StatusCode::FORBIDDEN, String::new()));
     }
 
-    // Save the file data locally
-    let file_name = format!("{}", layer.id);
+    // Save file locally
     let dir_path = Path::new("uploads");
-    let file_path = dir_path.join(&file_name);
-
+    let file_path = dir_path.join(&layer.id);
     fs::create_dir_all(dir_path)
         .await
         .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
@@ -88,15 +90,25 @@ pub async fn upload_layer<D: Database>(
         .await
         .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
 
+    // Process file and upload to PostGIS
     let postgis_uri = "postgresql://admin:password@localhost:5432/gridwalk";
-    let _processed_file = launch_process_file(file_path.to_str().unwrap(), &layer.id, postgis_uri);
+    launch_process_file(
+        file_path.to_str().unwrap(),
+        &layer.id,
+        postgis_uri,
+        &layer.workspace_id,
+    )
+    .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
     println!("Uploaded to POSTGIS!");
 
-    match layer.clone().write_record(state.app_data.clone()).await {
-        Ok(_) => {
-            let json_response = serde_json::to_value(&layer).unwrap();
-            Ok((StatusCode::OK, Json(json_response).to_string()))
-        }
-        Err(_) => Ok((StatusCode::INTERNAL_SERVER_ERROR, "".to_string())),
-    }
+    // Write layer record to database using Arc<dyn Database>
+    layer
+        .create(&state.app_data, user, &workspace)
+        .await
+        .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
+
+    // Return success response
+    let json_response =
+        serde_json::to_value(&layer).map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
+    Ok((StatusCode::OK, Json(json_response).to_string()))
 }
diff --git a/gridwalk-backend/src/routes/mod.rs b/gridwalk-backend/src/routes/mod.rs
@@ -1,10 +1,12 @@
 mod connector;
+mod layer;
 mod os_token;
 mod tiles;
 mod user;
 mod workspace;
 
 pub use connector::*;
+pub use layer::*;
 pub use os_token::*;
 pub use tiles::*;
 pub use user::*;

diff --git a/gridwalk-backend/src/server.rs b/gridwalk-backend/src/server.rs
@@ -2,7 +2,7 @@ use crate::app_state::AppState;
 use crate::auth::auth_middleware;
 use crate::routes::{
     add_workspace_member, create_connection, create_workspace, generate_os_token, health_check,
-    list_sources, login, logout, profile, register, remove_workspace_member, tiles,
+    list_sources, login, logout, profile, register, remove_workspace_member, tiles, upload_layer,
 };
 use axum::{
     middleware,
@@ -37,6 +37,7 @@ pub fn create_app(app_state: AppState) -> Router {
             "/workspace/:workspace_id/connection/:connection_id/sources",
             get(list_sources),
         )
+        .route("/upload_layer", post(upload_layer))
         .layer(middleware::from_fn_with_state(
             shared_state.clone(),
             auth_middleware,