0.1.8 RAM decrease when running many samples

CHANGELOG.md

@@ -1,3 +1,5 @@
+* v0.1.8
+	* RAM decrease when running many samples
 * v0.1.7
 	* Speedups and RAM decrease when using reference panels with fewer haplotypes
 * v0.1.6

QUILT.R

@@ -197,8 +197,8 @@ option_list <- list(
     make_option(
         "--record_read_label_usage",
         type = "logical",
-        help = "Whether to store what read labels were used during the Gibbs samplings (i.e. whether reads were assigned to arbitrary labelled haplotype 1 or 2) [default TRUE] ",
-        default = TRUE
+        help = "Whether to store what read labels were used during the Gibbs samplings (i.e. whether reads were assigned to arbitrary labelled haplotype 1 or 2) [default FALSE] ",
+        default = FALSE
     ), 
     make_option(
         "--record_interim_dosages",

QUILT/DESCRIPTION

@@ -1,8 +1,8 @@
 Package: QUILT
 Type: Package
 Title: QUILT
-Version: 0.1.7
-Date: 2021-04-22
+Version: 0.1.8
+Date: 2021-04-26
 Author: Robert William Davies
 Maintainer: Robert William Davies <[email protected]>
 Description: QUILT performs imputation of individuals sequenced to low coverage

QUILT/R/functions.R

@@ -608,8 +608,8 @@ get_and_impute_one_sample <- function(
     grid_has_read[wif0 + 1] <- TRUE
 
     nReads <- length(sampleReads)
-    super_out_hap_dosages <- as.list(1:nGibbsSamples)
-    super_out_read_labels <- as.list(1:nGibbsSamples)
+    ## super_out_hap_dosages <- as.list(1:nGibbsSamples)
+    ## super_out_read_labels <- as.list(1:nGibbsSamples)
     super_out_dosage_matrix <- as.list(1:nGibbsSamples)    
     read_label_matrix_all <- array(NA, c(nReads, nGibbsSamples)) ## need this for phasing - store final read labels in it
     read_label_matrix_conf <- array(FALSE, c(nReads, nGibbsSamples)) ## need this for phasing - store whether these are confident reads
@@ -993,7 +993,7 @@ get_and_impute_one_sample <- function(
                 minrp = 0.95,
                 minmp = 0.95
             )
-            super_out_hap_dosages[[i_gibbs_sample]] <- cbind(impute_all[["dosage1"]], impute_all[["dosage2"]])
+            ##super_out_hap_dosages[[i_gibbs_sample]] <- cbind(impute_all[["dosage1"]], impute_all[["dosage2"]])
             if (record_read_label_usage) {            
                 super_out_read_labels[[i_gibbs_sample]] <- read_label_matrix
             }
@@ -1145,23 +1145,24 @@ get_and_impute_one_sample <- function(
     ## what to return
     to_return <- list(
         sample_was_imputed = TRUE,
-        dosage = dosage,
-        super_out_hap_dosages = super_out_hap_dosages,
-        super_out_read_labels = super_out_read_labels,            
-        super_out_dosage_matrix = super_out_dosage_matrix,
-        phasing_haps = phasing_haps,
-        phasing_dosage = phasing_dosage,
-        pse_mat = pse_mat,
-        phasing_read_labels = phasing_read_labels,
-        imputed_truth_haplotypes = imputed_truth_haplotypes,
-        gp_t = gp_t,
         eij = eij,
         fij = fij,
         max_gen = max_gen,
         per_sample_alleleCount = per_sample_alleleCount,
         per_sample_vcf_col = per_sample_vcf_col
     )
 
+    ## phasing_read_labels = phasing_read_labels,
+    ## imputed_truth_haplotypes = imputed_truth_haplotypes,
+    ## pse_mat = pse_mat,
+    ## dosage = dosage,
+    ## super_out_dosage_matrix = super_out_dosage_matrix,
+    ## phasing_haps = phasing_haps,
+    ## phasing_dosage = phasing_dosage,
+    ## gp_t = gp_t,
+    ## super_out_hap_dosages = super_out_hap_dosages,
+    ## super_out_read_labels = super_out_read_labels,            
+
     if (hla_run) {
         ## print_message("More HLA SIMON code")
         to_return[["gamma1"]] <- gamma1

QUILT/R/quilt.R

@@ -101,7 +101,7 @@ QUILT <- function(
     verbose = TRUE,
     shuffle_bin_radius = 5000,
     iSizeUpperLimit = 1e6,
-    record_read_label_usage = TRUE,
+    record_read_label_usage = FALSE,
     record_interim_dosages = FALSE,
     use_bx_tag = TRUE,
     bxTagUpperLimit = 50000,
@@ -687,19 +687,30 @@ QUILT <- function(
                 use_small_eHapsCurrent_tc = use_small_eHapsCurrent_tc
             )
 
-            results_across_samples[[iSample - sampleRange[1] + 1]] <- out
-
             if (out[["sample_was_imputed"]]) {
                 ## for summarization
                 infoCount[, 1] <- infoCount[, 1, drop = FALSE] + out[["eij"]]
                 infoCount[, 2] <- infoCount[, 2, drop = FALSE] + (out[["fij"]] - out[["eij"]]**2)
                 afCount <- afCount + (out[["eij"]]) / 2
                 hweCount[out[["max_gen"]]] <- hweCount[out[["max_gen"]]] + 1 ## hmmmmm not ideal
                 alleleCount <- alleleCount + out[["per_sample_alleleCount"]]
+                ## drop now - not useful anymore
+                out[["eij"]] <- NULL
+                out[["fij"]] <- NULL
+                out[["per_sample_alleleCount"]] <- NULL
+                out[["max_gen"]] <- NULL
             }
 
+            results_across_samples[[iSample - sampleRange[1] + 1]] <- out
+
+            rm(out)
+
             ## optionally, do some gc here, if longer running job
             if (as.numeric(K) * as.numeric(nSNPs) > (1e6)) {
+                ## print("temporary")
+                ## print(head(sort( sapply(ls(),function(x){object.size(get(x))}), decreasing = TRUE)))
+                ## print(object.size(results_across_samples))
+                ## print(gc(reset = TRUE))
                 for(i in 1:5) {
                     gc(reset = TRUE)
                 }

README.md

@@ -1,7 +1,7 @@
 QUILT
 =====
-**__Current Version: 0.1.7__**
-Release date: April 22 2021
+**__Current Version: 0.1.8__**
+Release date: April 26 2021
 
 [![Build Status](https://img.shields.io/travis/rwdavies/QUILT/master.svg)](https://travis-ci.org/rwdavies/QUILT/)
 
@@ -49,8 +49,8 @@ git clone --recursive https://github.com/rwdavies/QUILT.git
 cd QUILT
 ./scripts/install-dependencies.sh
 cd releases
-wget https://github.com/rwdavies/quilt/releases/download/0.1.7/QUILT_0.1.7.tar.gz ## or curl -O
-R CMD INSTALL QUILT_0.1.7.tar.gz
+wget https://github.com/rwdavies/quilt/releases/download/0.1.8/QUILT_0.1.8.tar.gz ## or curl -O
+R CMD INSTALL QUILT_0.1.8.tar.gz
 ```
 
 ### conda <a name="paragraph-installation-conda"></a>