use qs for loading

Signed-off-by: Alex Pickering <[email protected]>
hms-dbmi-cellenics · Sep 25, 2024 · d4da131 · d4da131
1 parent de09ced
commit d4da131
Show file tree

Hide file tree

Showing 2 changed files with 77 additions and 2 deletions.
diff --git a/r/renv.lock b/r/renv.lock
@@ -439,6 +439,13 @@
       "Repository": "CRAN",
       "Hash": "d128ea05a972d3e67c6f39de52c72bd7"
     },
+    "RApiSerialize": {
+      "Package": "RApiSerialize",
+      "Version": "0.1.3",
+      "Source": "Repository",
+      "Repository": "CRAN",
+      "Hash": "12cbd319d03b55edaa195695a155bb30"
+    },
     "RColorBrewer": {
       "Package": "RColorBrewer",
       "Version": "1.1-3",
@@ -548,6 +555,16 @@
       ],
       "Hash": "1f2dc32c27746a35196aaf95adb357be"
     },
+    "RcppParallel": {
+      "Package": "RcppParallel",
+      "Version": "5.1.9",
+      "Source": "Repository",
+      "Repository": "CRAN",
+      "Requirements": [
+        "R"
+      ],
+      "Hash": "f38a72a419b91faac0ce5d9eee04c120"
+    },
     "RcppProgress": {
       "Package": "RcppProgress",
       "Version": "0.4.2",
@@ -2850,6 +2867,20 @@
       ],
       "Hash": "1cba04a4e9414bdefc9dcaa99649a8dc"
     },
+    "qs": {
+      "Package": "qs",
+      "Version": "0.26.3",
+      "Source": "Repository",
+      "Repository": "CRAN",
+      "Requirements": [
+        "BH",
+        "R",
+        "RApiSerialize",
+        "Rcpp",
+        "stringfish"
+      ],
+      "Hash": "e372cf3f70e3c7a768ec5ea14f0f3edd"
+    },
     "ragg": {
       "Package": "ragg",
       "Version": "1.3.2",
@@ -3509,6 +3540,18 @@
       ],
       "Hash": "26e158d12052c279bdd4ba858b80fb1f"
     },
+    "stringfish": {
+      "Package": "stringfish",
+      "Version": "0.16.0",
+      "Source": "Repository",
+      "Repository": "CRAN",
+      "Requirements": [
+        "R",
+        "Rcpp",
+        "RcppParallel"
+      ],
+      "Hash": "b7eb79470319ae71d4b5ed9cd7bf7294"
+    },
     "stringi": {
       "Package": "stringi",
       "Version": "1.8.4",

diff --git a/r/work.R b/r/work.R
@@ -7,6 +7,16 @@ options(Seurat.object.assay.version = "v5")
 for (f in list.files("R", ".R$", full.names = TRUE)) source(f)
 load('R/sysdata.rda') # constants
 
+readFlex <- function(fpath) {
+
+  if (tools::file_ext(fpath) == 'qs') {
+    data <- qs::qread(fpath)
+  } else {
+    data <- readRDS(fpath)
+  }
+  return(data)
+}
+
 load_data <- function(fpath) {
   loaded <- FALSE
   data <- NULL
@@ -18,7 +28,7 @@ load_data <- function(fpath) {
         print(getwd())
         print("Experiment folder status:")
         print(list.files(dirname(fpath), all.files = TRUE, full.names = TRUE))
-        f <- readRDS(fpath)
+        f <- readFlex(fpath)
         loaded <- TRUE
         length <- dim(f)
 
@@ -316,12 +326,34 @@ repeat {
 }
 
 backend <- RestRserve::BackendRserve$new()
-fpath <- file.path("/data", experiment_id, "r.rds")
+
+get_fpath <- function(experiment_id) {
+  data_dir <- file.path('/data', experiment_id)
+
+  fpath <- NULL
+
+  while (is.null(fpath)) {
+    fnames <- list.files(data_dir)
+
+    if ('r.qs' %in% fnames) {
+      fpath <- file.path(data_dir, 'r.qs')
+
+    } else if ('r.rds' %in% fnames) {
+      # fallback to rds for older experiments
+      fpath <- file.path(data_dir, 'r.rds')
+    }
+    Sys.sleep(1)
+  }
+
+  return(fpath)
+}
 
 repeat {
   # need to load here as can change e.g. integration method
   cleanupMarkersCache()
 
+  fpath <- get_fpath(experiment_id)
+
   data <- load_data(fpath)
   last_modified <- file.info(fpath)$mtime
   app <- create_app(last_modified, data, fpath)