From 641c4f0736b526848a5c552837cf14dec8d395cd Mon Sep 17 00:00:00 2001 From: Chris Carlon Date: Thu, 31 Oct 2024 16:16:35 +0000 Subject: [PATCH] fix(backend and frontend): Continuation of backend and frontend debugging when uploading files above 4mb [2024-10-31] --- gridwalk-backend/src/routes/layer.rs | 288 ++++++++++++++---- .../src/app/api/upload/layer/config.ts | 6 + gridwalk-ui/src/app/api/upload/layer/route.ts | 174 ++++++++--- gridwalk-ui/src/app/project/page.tsx | 46 +-- 4 files changed, 395 insertions(+), 119 deletions(-) create mode 100644 gridwalk-ui/src/app/api/upload/layer/config.ts diff --git a/gridwalk-backend/src/routes/layer.rs b/gridwalk-backend/src/routes/layer.rs index eacb19c..c25a362 100644 --- a/gridwalk-backend/src/routes/layer.rs +++ b/gridwalk-backend/src/routes/layer.rs @@ -1,12 +1,13 @@ use crate::app_state::AppState; use crate::auth::AuthUser; -use crate::core::{CreateLayer, Layer, Workspace, WorkspaceRole}; +use crate::core::{CreateLayer, Layer, User, Workspace, WorkspaceRole}; use axum::{ extract::{Extension, Multipart, State}, - http::StatusCode, + http::{HeaderMap, StatusCode}, response::IntoResponse, Json, }; +use serde_json::json; use std::path::Path; use std::sync::Arc; use tokio::{ @@ -17,96 +18,249 @@ use tokio::{ pub async fn upload_layer( State(state): State>, Extension(auth_user): Extension, + headers: HeaderMap, mut multipart: Multipart, -) -> Result { - // Early return if no user - let user = auth_user.user.as_ref().ok_or(StatusCode::UNAUTHORIZED)?; - let mut file_data = Vec::new(); +) -> Result)> { + // Extract headers + let file_type = headers + .get("x-file-type") + .and_then(|v| v.to_str().ok()) + .unwrap_or(""); + + let workspace_id = headers + .get("x-workspace-id") + .and_then(|v| v.to_str().ok()) + .ok_or_else(|| { + let error = json!({ + "error": "Missing workspace ID in headers", + "details": null + }); + (StatusCode::BAD_REQUEST, Json(error)) + })?; + + tracing::info!( + "Processing upload for file type: {} workspace: {}", + file_type, + workspace_id + ); + + let user = auth_user.user.as_ref().ok_or_else(|| { + let error = json!({ + "error": "Unauthorized request", + "details": null + }); + (StatusCode::UNAUTHORIZED, Json(error)) + })?; + let mut layer_info: Option = None; + let mut file_path = None; - // Process multipart form data - while let Some(field) = multipart - .next_field() - .await - .map_err(|_| StatusCode::BAD_REQUEST)? - { - let name = field.name().ok_or(StatusCode::BAD_REQUEST)?.to_string(); - match name.as_str() { - "file" => { - file_data = field - .bytes() - .await - .map_err(|_| StatusCode::BAD_REQUEST)? - .to_vec(); - println!("File data size: {} bytes", file_data.len()); - } - "layer_info" => { - let json_str = String::from_utf8( - field - .bytes() - .await - .map_err(|_| StatusCode::BAD_REQUEST)? - .to_vec(), - ) - .map_err(|_| StatusCode::BAD_REQUEST)?; - layer_info = - Some(serde_json::from_str(&json_str).map_err(|_| StatusCode::BAD_REQUEST)?); + // Create uploads directory + let dir_path = Path::new("uploads"); + fs::create_dir_all(dir_path).await.map_err(|e| { + let error = json!({ + "error": "Failed to create uploads directory", + "details": e.to_string() + }); + (StatusCode::INTERNAL_SERVER_ERROR, Json(error)) + })?; + + // Process multipart form + while let Some(field) = multipart.next_field().await.map_err(|e| { + let error = json!({ + "error": "Failed to process form field", + "details": e.to_string() + }); + (StatusCode::BAD_REQUEST, Json(error)) + })? { + if let Some(name) = field.name() { + tracing::info!("Processing field: {}", name); + + match name { + "file" => { + if let Some(filename) = field.file_name() { + tracing::info!("Processing file: {}", filename); + + let temp_path = + dir_path.join(format!("temp_{}_{}", uuid::Uuid::new_v4(), filename)); + + let mut file = File::create(&temp_path).await.map_err(|e| { + let error = json!({ + "error": "Failed to create temporary file", + "details": e.to_string() + }); + (StatusCode::INTERNAL_SERVER_ERROR, Json(error)) + })?; + + let mut total_bytes = 0usize; + + // Stream the file + let mut field = field; + while let Some(chunk) = field.chunk().await.map_err(|e| { + let error = json!({ + "error": "Failed to read file chunk", + "details": e.to_string() + }); + (StatusCode::BAD_REQUEST, Json(error)) + })? { + total_bytes += chunk.len(); + file.write_all(&chunk).await.map_err(|e| { + let error = json!({ + "error": "Failed to write file chunk", + "details": e.to_string() + }); + (StatusCode::INTERNAL_SERVER_ERROR, Json(error)) + })?; + } + + file.sync_all().await.map_err(|e| { + let error = json!({ + "error": "Failed to sync file to disk", + "details": e.to_string() + }); + (StatusCode::INTERNAL_SERVER_ERROR, Json(error)) + })?; + + tracing::info!("File upload complete: {} bytes", total_bytes); + file_path = Some(temp_path); + } + } + "layer_info" => { + let bytes = field.bytes().await.map_err(|e| { + let error = json!({ + "error": "Failed to read layer info", + "details": e.to_string() + }); + (StatusCode::BAD_REQUEST, Json(error)) + })?; + + let info_str = String::from_utf8(bytes.to_vec()).map_err(|e| { + let error = json!({ + "error": "Invalid UTF-8 in layer info", + "details": e.to_string() + }); + (StatusCode::BAD_REQUEST, Json(error)) + })?; + + tracing::debug!("Received layer info: {}", info_str); + + layer_info = Some(serde_json::from_str(&info_str).map_err(|e| { + let error = json!({ + "error": "Invalid layer info JSON", + "details": e.to_string() + }); + (StatusCode::BAD_REQUEST, Json(error)) + })?); + } + _ => { + tracing::warn!("Unexpected field: {}", name); + } } - _ => {} } } - let layer_info = layer_info.ok_or(StatusCode::BAD_REQUEST)?; + // Validate required data + let final_path = file_path.ok_or_else(|| { + let error = json!({ + "error": "No file was uploaded", + "details": null + }); + (StatusCode::BAD_REQUEST, Json(error)) + })?; + + let layer_info = layer_info.ok_or_else(|| { + let error = json!({ + "error": "No layer info provided", + "details": null + }); + (StatusCode::BAD_REQUEST, Json(error)) + })?; + + // Create and process layer let layer = Layer::from_req(layer_info, user); - // Get workspace and check permissions + // Handle the rest of the process with proper error responses + match process_layer(&state, &layer, user, &final_path).await { + Ok(json_response) => Ok((StatusCode::OK, Json(json_response))), + Err(e) => { + if let Err(cleanup_err) = fs::remove_file(&final_path).await { + tracing::error!("Failed to clean up file after error: {}", cleanup_err); + } + Err(e) + } + } +} + +async fn process_layer( + state: &Arc, + layer: &Layer, + user: &User, + file_path: &Path, +) -> Result)> { + // Validate workspace access let workspace = Workspace::from_id(&state.app_data, &layer.workspace_id) .await - .map_err(|_| StatusCode::NOT_FOUND)?; + .map_err(|e| { + let error = json!({ + "error": "Workspace not found", + "details": e.to_string() + }); + (StatusCode::NOT_FOUND, Json(error)) + })?; let member = workspace - .get_member(&state.app_data, &user) + .get_member(&state.app_data, user) .await - .map_err(|_| StatusCode::FORBIDDEN)?; + .map_err(|e| { + let error = json!({ + "error": "Access forbidden", + "details": e.to_string() + }); + (StatusCode::FORBIDDEN, Json(error)) + })?; if member.role == WorkspaceRole::Read { - return Ok((StatusCode::FORBIDDEN, String::new())); + let error = json!({ + "error": "Read-only access", + "details": "User does not have write permission" + }); + return Err((StatusCode::FORBIDDEN, Json(error))); } - // Save file locally - let dir_path = Path::new("uploads"); - let file_path = dir_path.join(&layer.name); - let file_path_str = file_path.to_str().unwrap(); - - fs::create_dir_all(dir_path) - .await - .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; - - let mut file = File::create(&file_path) - .await - .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; - - file.write_all(&file_data) - .await - .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; - - // Check permissions of user + // Create layer in database layer .create(&state.app_data, user, &workspace) .await - .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + .map_err(|e| { + let error = json!({ + "error": "Failed to create layer", + "details": e.to_string() + }); + (StatusCode::INTERNAL_SERVER_ERROR, Json(error)) + })?; - // Write layer table to PostGis + // Process the file layer - .send_to_postgis(file_path_str) + .send_to_postgis(file_path.to_str().unwrap()) .await - .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + .map_err(|e| { + let error = json!({ + "error": "Failed to process file", + "details": e.to_string() + }); + (StatusCode::INTERNAL_SERVER_ERROR, Json(error)) + })?; - // Write layer record to App Database - layer.write_record(&state.app_data).await.ok(); + // Write the record + if let Err(e) = layer.write_record(&state.app_data).await { + tracing::error!("Failed to write layer record: {}", e); + } // Return success response - let json_response = - serde_json::to_value(&layer).map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; - Ok((StatusCode::OK, Json(json_response).to_string())) + Ok(serde_json::to_value(layer).unwrap_or_else(|_| { + json!({ + "status": "success", + "message": "Layer created successfully" + }) + })) } diff --git a/gridwalk-ui/src/app/api/upload/layer/config.ts b/gridwalk-ui/src/app/api/upload/layer/config.ts new file mode 100644 index 0000000..4c5e4a0 --- /dev/null +++ b/gridwalk-ui/src/app/api/upload/layer/config.ts @@ -0,0 +1,6 @@ +export const config = { + api: { + bodyParser: false, + sizeLimit: "50mb", + }, +}; diff --git a/gridwalk-ui/src/app/api/upload/layer/route.ts b/gridwalk-ui/src/app/api/upload/layer/route.ts index 14b528d..29b2082 100644 --- a/gridwalk-ui/src/app/api/upload/layer/route.ts +++ b/gridwalk-ui/src/app/api/upload/layer/route.ts @@ -1,3 +1,4 @@ +// gridwalk-ui/src/app/api/upload/layer/route.ts import { NextRequest } from "next/server"; const HARDCODED_AUTH_TOKEN = "VVPME0BYEDG7LJYGLL9PKJ8AS1GABM"; @@ -5,69 +6,172 @@ const HARDCODED_AUTH_TOKEN = "VVPME0BYEDG7LJYGLL9PKJ8AS1GABM"; interface LayerInfo { workspace_id: string; name: string; + file_type?: string; +} + +interface FileConfig { + maxSize: number; + contentType: string; + streamable: boolean; + originalType?: string; +} + +type FileConfigs = { + readonly [key: string]: FileConfig; +}; + +const FILE_CONFIGS: FileConfigs = { + ".geojson": { + maxSize: 50 * 1024 * 1024, + contentType: "application/geo+json", + streamable: false, + }, + ".json": { + maxSize: 50 * 1024 * 1024, + contentType: "application/json", + streamable: false, + }, + ".kml": { + maxSize: 50 * 1024 * 1024, + contentType: "application/vnd.google-earth.kml+xml", + streamable: false, + }, + ".shp": { + maxSize: 100 * 1024 * 1024, + contentType: "application/x-esri-shape", + streamable: true, + }, + ".gpkg": { + maxSize: 500 * 1024 * 1024, + contentType: "application/octet-stream", + originalType: "application/geopackage+sqlite3", + streamable: true, + }, +} as const; + +async function prepareUploadFormData( + file: File, + workspaceId: string, + layerName?: string, +): Promise<{ formData: FormData; contentType: string; originalType?: string }> { + const extension = "." + file.name.split(".").pop()?.toLowerCase(); + const config = FILE_CONFIGS[extension]; + + if (!config) { + throw new Error(`Unsupported file type: ${extension}`); + } + + const formData = new FormData(); + + // Handle the file + if (extension === ".gpkg") { + const arrayBuffer = await file.arrayBuffer(); + const blob = new Blob([arrayBuffer], { + type: "application/octet-stream", + }); + formData.append("file", blob, file.name); + } else { + formData.append("file", file); + } + + // Append layer info as a separate part + const layerInfo: LayerInfo = { + workspace_id: workspaceId, + name: layerName ?? file.name, + file_type: extension.substring(1), + }; + + // Important: Use proper JSON stringification + formData.append("layer_info", JSON.stringify(layerInfo)); + + // Log the form data contents for debugging + console.log("Form data contents:", { + layerInfo, + fileName: file.name, + fileSize: file.size, + }); + + return { + formData, + contentType: config.contentType, + originalType: config.originalType, + }; } export async function POST(request: NextRequest) { try { const formData = await request.formData(); + const file = formData.get("file") as File | null; + const workspaceId = formData.get("workspace_id") as string | null; + const layerName = formData.get("name") as string | null; - // Validation step - const file = formData.get("file"); - const workspaceId = formData.get("workspace_id"); - const layerName = formData.get("name"); - - // Early validation return - if (!file || !(file instanceof File)) { + if (!file || !workspaceId) { return Response.json( - { success: false, error: "No file provided" }, + { + success: false, + error: + `Missing required fields: ${!file ? "file" : ""} ${!workspaceId ? "workspace_id" : ""}`.trim(), + }, { status: 400 }, ); } - if (!workspaceId || !layerName) { - return Response.json( - { success: false, error: "Missing workspace_id or name" }, - { status: 400 }, - ); - } + console.log("Received upload request:", { + fileName: file.name, + fileSize: file.size, + workspaceId, + layerName, + }); + + const { + formData: apiFormData, + contentType, + originalType, + } = await prepareUploadFormData(file, workspaceId, layerName || undefined); - // Transform for Rust backend - const layerInfo: LayerInfo = { - workspace_id: workspaceId.toString(), - name: layerName.toString(), + const headers: HeadersInit = { + Authorization: `Bearer ${HARDCODED_AUTH_TOKEN}`, + Accept: "application/json", + "X-File-Type": "." + file.name.split(".").pop()?.toLowerCase(), + "X-Original-Content-Type": originalType || contentType, + "X-Workspace-Id": workspaceId, }; - const apiFormData = new FormData(); - apiFormData.append("file", file); - apiFormData.append( - "layer_info", - new Blob([JSON.stringify(layerInfo)], { type: "application/json" }), - ); + console.log("Sending request to backend:", { + headers, + formDataEntries: Array.from(apiFormData.entries()).map(([key]) => key), + }); - // Send to backend const response = await fetch("http://localhost:3001/upload_layer", { method: "POST", - headers: { - Authorization: `Bearer ${HARDCODED_AUTH_TOKEN}`, - }, + headers, body: apiFormData, }); + const responseText = await response.text(); + console.log("Raw backend response:", responseText); + if (!response.ok) { - const errorText = await response.text(); - throw new Error(errorText || response.statusText); + throw new Error(responseText || `Backend error: ${response.status}`); } - console.log("SUCCESS"); - - const data = await response.json(); + const data = responseText ? JSON.parse(responseText) : null; return Response.json({ success: true, data }); } catch (error) { - console.error("Upload error:", error); + console.error("Upload error:", { + error, + message: error instanceof Error ? error.message : "Unknown error", + stack: error instanceof Error ? error.stack : undefined, + }); + return Response.json( { success: false, - error: error instanceof Error ? error.message : "Upload failed", + error: + error instanceof Error + ? error.message + : "Internal server error during upload", + details: error instanceof Error ? error.stack : undefined, }, { status: 500 }, ); diff --git a/gridwalk-ui/src/app/project/page.tsx b/gridwalk-ui/src/app/project/page.tsx index 164ff24..005c2eb 100644 --- a/gridwalk-ui/src/app/project/page.tsx +++ b/gridwalk-ui/src/app/project/page.tsx @@ -71,6 +71,7 @@ export default function Project() { }); // Upload handler with improved error handling and type safety + // In your Project component const handleLayerUpload = useCallback( async ( file: File, @@ -78,20 +79,21 @@ export default function Project() { ): Promise => { setIsUploading(true); setError(null); - setUploadSuccess(false); - if (!file) { - setError("No file provided"); + // Add client-side validation + const MAX_SIZE = 50 * 1024 * 1024; // 50MB + if (file.size > MAX_SIZE) { + setError("File size exceeds 50MB limit"); setIsUploading(false); return; } - try { - const formData = new FormData(); - formData.append("file", file); - formData.append("workspace_id", workspaceId); - formData.append("name", file.name); + const formData = new FormData(); + formData.append("file", file); + formData.append("workspace_id", workspaceId); + formData.append("name", file.name); + try { const response = await fetch("/api/upload/layer", { method: "POST", body: formData, @@ -100,25 +102,35 @@ export default function Project() { const data = (await response.json()) as UploadResponse; if (!response.ok || !data.success) { - throw new Error( - data.error || `Upload failed with status: ${response.status}`, - ); + throw new Error(data.error || `Upload failed: ${response.status}`); } - // Set success and reset after 3 seconds + + // Update layers state + if (data.data) { + setLayers((prev) => [ + ...prev, + { + id: data.data!.id, + name: data.data!.name, + type: file.type, + visible: true, + workspace_id: data.data!.workspace_id, + }, + ]); + } + setUploadSuccess(true); setTimeout(() => setUploadSuccess(false), 3000); } catch (err) { const errorMessage = - err instanceof Error - ? err.message - : "Unknown error during file upload"; + err instanceof Error ? err.message : "Unknown error during upload"; setError(errorMessage); - console.error("Layer upload error:", err); + console.error("Upload error:", err); } finally { setIsUploading(false); } }, - [], + [setLayers], ); // Event handlers with proper typing