Skip to content

Commit

Permalink
Merge pull request #322 from hms-dbmi-cellenics/fast-build
Browse files Browse the repository at this point in the history 
implement fast restore
  • Loading branch information
@alexvpickering
alexvpickering authored Aug 17, 2023
2 parents fb7ab4b + 56fe1b5 commit 5345e5f
Show file tree
Hide file tree
Showing 6 changed files with 322 additions and 23 deletions.
3 changes: 2 additions & 1 deletion pipeline-runner/DESCRIPTION
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,8 @@ Suggests:
styler,
testthat (>= 3.0.0),
usethis,
glmGamPoi
glmGamPoi,
hdf5r
Config/testthat/edition: 3
Encoding: UTF-8
LazyData: true
Expand Down
27 changes: 13 additions & 14 deletions pipeline-runner/Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,27 +13,28 @@ ARG GITHUB_PAT
ENV GITHUB_PAT $GITHUB_PAT
RUN R -q -e "if(Sys.getenv('GITHUB_PAT') == '') stop('need to export GITHUB_PAT')"

# renv library
ENV RENV_LIB=/src/lib

# add renv library to .libPaths
# this is path with R package folders when renv deactivated or not used
ENV RENV_LIB=/src/lib
RUN echo ".libPaths(c('$RENV_LIB', .libPaths()))" >> $(R RHOME)/etc/Rprofile.site

# install renv to install required R packages
RUN R -q -e "install.packages('remotes', repos = c(CRAN = 'https://cloud.r-project.org'))" && \
R -q -e "remotes::install_github('rstudio/[email protected]')" && \
R -q -e "renv::init(bare = TRUE, settings = list(use.cache = FALSE))"

# fast pre-restore with pkgbuild
# an initial lockfile is used to avoid frequent re-installs
# use renv::snapshot(lockfile='renv.lock.init') if R dependency updates become slow to build
# delete renv cache
# strip debug from shared libraries
# see http://dirk.eddelbuettel.com/blog/2017/08/20/#010_stripping_shared_libraries
COPY ./renv.lock.init .
RUN R -q -e "renv::restore(lockfile='renv.lock.init', library = '$RENV_LIB')" && \
R -q -e 'root <- renv::paths$root(); unlink(root, recursive = TRUE)' && \
strip --strip-debug $RENV_LIB/*/libs/*.so
# copy renv.lock.init to renv.lock if R dependency updates become slow to build
COPY ./renv.lock.init ./renv.lock
COPY setup/restore_fast.R .
RUN Rscript restore_fast.R

# restore anything pkgbuild missed
COPY setup/restore_renv.R .
RUN Rscript restore_renv.R

# deactivate so that .libPaths used (set above)
RUN R -q -e "renv::deactivate()"

# install miniconda and geosketch
Expand All @@ -46,9 +47,7 @@ RUN R -q -e "reticulate::install_miniconda()" && \

# use renv::snapshot() while R dependency updates are quick to build
COPY ./renv.lock .
RUN R -q -e "renv::restore(lockfile='renv.lock', library = '$RENV_LIB', clean = TRUE)" && \
R -q -e 'root <- renv::paths$root(); unlink(root, recursive = TRUE)' && \
strip --strip-debug $RENV_LIB/*/libs/*.so
RUN Rscript restore_renv.R

# determine system run-time deps
COPY setup/get_sysdeps_run.R .
Expand Down
102 changes: 98 additions & 4 deletions pipeline-runner/renv.lock
View file
Original file line number Diff line number Diff line change
Expand Up @@ -993,10 +993,10 @@
},
"cli": {
"Package": "cli",
"Version": "3.4.1",
"Version": "3.6.1",
"Source": "Repository",
"Repository": "CRAN",
"Hash": "0d297d01734d2bcea40197bd4971a764",
"Hash": "89e6d8219950eac806ae0c489052048a",
"Requirements": []
},
"cluster": {
Expand Down Expand Up @@ -1232,6 +1232,14 @@
"Hash": "77bd60a6157420d4ffa93b27cf6a58b8",
"Requirements": []
},
"filelock": {
"Package": "filelock",
"Version": "1.0.2",
"Source": "Repository",
"Repository": "CRAN",
"Hash": "38ec653c2613bed60052ba3787bd8a2c",
"Requirements": []
},
"fitdistrplus": {
"Package": "fitdistrplus",
"Version": "1.1-5",
Expand Down Expand Up @@ -1740,6 +1748,14 @@
"lattice"
]
},
"lpSolve": {
"Package": "lpSolve",
"Version": "5.6.18",
"Source": "Repository",
"Repository": "CRAN",
"Hash": "12c7a918599d5700e4265dd8a21f684f",
"Requirements": []
},
"magrittr": {
"Package": "magrittr",
"Version": "2.0.3",
Expand Down Expand Up @@ -2049,6 +2065,41 @@
"vctrs"
]
},
"pkgbuild": {
"Package": "pkgbuild",
"Version": "1.4.2",
"Source": "Repository",
"Repository": "CRAN",
"Hash": "beb25b32a957a22a5c301a9e441190b3",
"Requirements": [
"R6",
"callr",
"cli",
"crayon",
"desc",
"prettyunits",
"processx",
"rprojroot"
]
},
"pkgcache": {
"Package": "pkgcache",
"Version": "2.2.0.1",
"Source": "Repository",
"Repository": "CRAN",
"Hash": "2622106d5651a653c55430ee5f583978",
"Requirements": [
"R6",
"callr",
"cli",
"curl",
"filelock",
"jsonlite",
"prettyunits",
"processx",
"rappdirs"
]
},
"pkgconfig": {
"Package": "pkgconfig",
"Version": "2.0.3",
Expand All @@ -2057,6 +2108,31 @@
"Hash": "01f28d4278f15c76cddbea05899c5d6f",
"Requirements": []
},
"pkgdepends": {
"Package": "pkgdepends",
"Version": "0.6.0",
"Source": "Repository",
"Repository": "CRAN",
"Hash": "d0a9c1ca3fbc82d3db84e65f09b42720",
"Requirements": [
"R6",
"callr",
"cli",
"curl",
"desc",
"filelock",
"glue",
"jsonlite",
"lpSolve",
"pkgbuild",
"pkgcache",
"prettyunits",
"processx",
"ps",
"rprojroot",
"zip"
]
},
"pkgload": {
"Package": "pkgload",
"Version": "1.3.2",
Expand Down Expand Up @@ -2138,6 +2214,14 @@
"Hash": "a555924add98c99d2f411e37e7d25e9f",
"Requirements": []
},
"prettyunits": {
"Package": "prettyunits",
"Version": "1.1.1",
"Source": "Repository",
"Repository": "CRAN",
"Hash": "95ef9167b75dde9d2ccc3c7528393e7e",
"Requirements": []
},
"processx": {
"Package": "processx",
"Version": "3.8.0",
Expand Down Expand Up @@ -2660,9 +2744,11 @@
"spatstat.core": {
"Package": "spatstat.core",
"Version": "2.2-0",
"Source": "Repository",
"Source": "URL",
"Repository": "CRAN",
"Hash": "46bda789e91bae756a29fd2cc7ead800",
"RemoteType": "url",
"RemoteUrl": "https://cran.r-project.org/src/contrib/Archive/spatstat.core/spatstat.core_2.2-0.tar.gz",
"Hash": "8ef00c4ad211bd55a52a74c4b3388b40",
"Requirements": [
"Matrix",
"abind",
Expand Down Expand Up @@ -3004,6 +3090,14 @@
"Hash": "ee9b643aa8331c45d8d82eb3a137c9bc",
"Requirements": []
},
"zip": {
"Package": "zip",
"Version": "2.3.0",
"Source": "Repository",
"Repository": "CRAN",
"Hash": "d98c94dacb7e0efcf83b0a133a705504",
"Requirements": []
},
"zlibbioc": {
"Package": "zlibbioc",
"Version": "1.42.0",
Expand Down
102 changes: 98 additions & 4 deletions pipeline-runner/renv.lock.init
View file
Original file line number Diff line number Diff line change
Expand Up @@ -993,10 +993,10 @@
},
"cli": {
"Package": "cli",
"Version": "3.4.1",
"Version": "3.6.1",
"Source": "Repository",
"Repository": "CRAN",
"Hash": "0d297d01734d2bcea40197bd4971a764",
"Hash": "89e6d8219950eac806ae0c489052048a",
"Requirements": []
},
"cluster": {
Expand Down Expand Up @@ -1232,6 +1232,14 @@
"Hash": "77bd60a6157420d4ffa93b27cf6a58b8",
"Requirements": []
},
"filelock": {
"Package": "filelock",
"Version": "1.0.2",
"Source": "Repository",
"Repository": "CRAN",
"Hash": "38ec653c2613bed60052ba3787bd8a2c",
"Requirements": []
},
"fitdistrplus": {
"Package": "fitdistrplus",
"Version": "1.1-5",
Expand Down Expand Up @@ -1740,6 +1748,14 @@
"lattice"
]
},
"lpSolve": {
"Package": "lpSolve",
"Version": "5.6.18",
"Source": "Repository",
"Repository": "CRAN",
"Hash": "12c7a918599d5700e4265dd8a21f684f",
"Requirements": []
},
"magrittr": {
"Package": "magrittr",
"Version": "2.0.3",
Expand Down Expand Up @@ -2049,6 +2065,41 @@
"vctrs"
]
},
"pkgbuild": {
"Package": "pkgbuild",
"Version": "1.4.2",
"Source": "Repository",
"Repository": "CRAN",
"Hash": "beb25b32a957a22a5c301a9e441190b3",
"Requirements": [
"R6",
"callr",
"cli",
"crayon",
"desc",
"prettyunits",
"processx",
"rprojroot"
]
},
"pkgcache": {
"Package": "pkgcache",
"Version": "2.2.0.1",
"Source": "Repository",
"Repository": "CRAN",
"Hash": "2622106d5651a653c55430ee5f583978",
"Requirements": [
"R6",
"callr",
"cli",
"curl",
"filelock",
"jsonlite",
"prettyunits",
"processx",
"rappdirs"
]
},
"pkgconfig": {
"Package": "pkgconfig",
"Version": "2.0.3",
Expand All @@ -2057,6 +2108,31 @@
"Hash": "01f28d4278f15c76cddbea05899c5d6f",
"Requirements": []
},
"pkgdepends": {
"Package": "pkgdepends",
"Version": "0.6.0",
"Source": "Repository",
"Repository": "CRAN",
"Hash": "d0a9c1ca3fbc82d3db84e65f09b42720",
"Requirements": [
"R6",
"callr",
"cli",
"curl",
"desc",
"filelock",
"glue",
"jsonlite",
"lpSolve",
"pkgbuild",
"pkgcache",
"prettyunits",
"processx",
"ps",
"rprojroot",
"zip"
]
},
"pkgload": {
"Package": "pkgload",
"Version": "1.3.2",
Expand Down Expand Up @@ -2138,6 +2214,14 @@
"Hash": "a555924add98c99d2f411e37e7d25e9f",
"Requirements": []
},
"prettyunits": {
"Package": "prettyunits",
"Version": "1.1.1",
"Source": "Repository",
"Repository": "CRAN",
"Hash": "95ef9167b75dde9d2ccc3c7528393e7e",
"Requirements": []
},
"processx": {
"Package": "processx",
"Version": "3.8.0",
Expand Down Expand Up @@ -2660,9 +2744,11 @@
"spatstat.core": {
"Package": "spatstat.core",
"Version": "2.2-0",
"Source": "Repository",
"Source": "URL",
"Repository": "CRAN",
"Hash": "46bda789e91bae756a29fd2cc7ead800",
"RemoteType": "url",
"RemoteUrl": "https://cran.r-project.org/src/contrib/Archive/spatstat.core/spatstat.core_2.2-0.tar.gz",
"Hash": "8ef00c4ad211bd55a52a74c4b3388b40",
"Requirements": [
"Matrix",
"abind",
Expand Down Expand Up @@ -3004,6 +3090,14 @@
"Hash": "ee9b643aa8331c45d8d82eb3a137c9bc",
"Requirements": []
},
"zip": {
"Package": "zip",
"Version": "2.3.0",
"Source": "Repository",
"Repository": "CRAN",
"Hash": "d98c94dacb7e0efcf83b0a133a705504",
"Requirements": []
},
"zlibbioc": {
"Package": "zlibbioc",
"Version": "1.42.0",
Expand Down
Loading

0 comments on commit 5345e5f

Please sign in to comment.