use qs for uploading matrix

Signed-off-by: Alex Pickering <[email protected]>
hms-dbmi-cellenics · Sep 25, 2024 · 58894a6 · 58894a6
1 parent ab94ad0
commit 58894a6
Show file tree

Hide file tree

Showing 2 changed files with 47 additions and 3 deletions.
diff --git a/pipeline-runner/R/handle_data.R b/pipeline-runner/R/handle_data.R
@@ -337,12 +337,13 @@ send_plot_data_to_s3 <- function(pipeline_config, experiment_id, output) {
 }
 
 upload_matrix_to_s3 <- function(pipeline_config, experiment_id, data) {
-  object_key <- paste0(experiment_id, "/r.rds")
+  object_key <- paste0(experiment_id, "/r.qs")
 
+  message("Saving object for key: ", object_key, '...')
   count_matrix <- tempfile()
-  saveRDS(data, file = count_matrix)
+  qs::qsave(data, file = count_matrix)
 
-  message("Count matrix file size : ", format(object.size(count_matrix)))
+  message("Count matrix file size : ", fs::file_size(count_matrix))
   message("Uploading updated count matrix to S3 bucket ", pipeline_config$processed_bucket, " at key ", object_key, "...")
 
   put_object_in_s3_multipart(pipeline_config, pipeline_config$processed_bucket, count_matrix, object_key)

diff --git a/pipeline-runner/renv.lock b/pipeline-runner/renv.lock
@@ -484,6 +484,13 @@
       "Repository": "CRAN",
       "Hash": "d128ea05a972d3e67c6f39de52c72bd7"
     },
+    "RApiSerialize": {
+      "Package": "RApiSerialize",
+      "Version": "0.1.3",
+      "Source": "Repository",
+      "Repository": "CRAN",
+      "Hash": "12cbd319d03b55edaa195695a155bb30"
+    },
     "RColorBrewer": {
       "Package": "RColorBrewer",
       "Version": "1.1-3",
@@ -619,6 +626,16 @@
       ],
       "Hash": "225157373f361daf85198a8d1ddaa733"
     },
+    "RcppParallel": {
+      "Package": "RcppParallel",
+      "Version": "5.1.9",
+      "Source": "Repository",
+      "Repository": "CRAN",
+      "Requirements": [
+        "R"
+      ],
+      "Hash": "f38a72a419b91faac0ce5d9eee04c120"
+    },
     "RcppProgress": {
       "Package": "RcppProgress",
       "Version": "0.4.2",
@@ -2850,6 +2867,20 @@
       ],
       "Hash": "1cba04a4e9414bdefc9dcaa99649a8dc"
     },
+    "qs": {
+      "Package": "qs",
+      "Version": "0.26.3",
+      "Source": "Repository",
+      "Repository": "CRAN",
+      "Requirements": [
+        "BH",
+        "R",
+        "RApiSerialize",
+        "Rcpp",
+        "stringfish"
+      ],
+      "Hash": "e372cf3f70e3c7a768ec5ea14f0f3edd"
+    },
     "ragg": {
       "Package": "ragg",
       "Version": "1.3.2",
@@ -3508,6 +3539,18 @@
       ],
       "Hash": "26e158d12052c279bdd4ba858b80fb1f"
     },
+    "stringfish": {
+      "Package": "stringfish",
+      "Version": "0.16.0",
+      "Source": "Repository",
+      "Repository": "CRAN",
+      "Requirements": [
+        "R",
+        "Rcpp",
+        "RcppParallel"
+      ],
+      "Hash": "b7eb79470319ae71d4b5ed9cd7bf7294"
+    },
     "stringi": {
       "Package": "stringi",
       "Version": "1.8.4",