diff --git a/vignettes/collaboration.Rmd b/vignettes/collaboration.Rmd index 7d384221..8e2f9732 100644 --- a/vignettes/collaboration.Rmd +++ b/vignettes/collaboration.Rmd @@ -8,23 +8,8 @@ vignette: > --- ```{r, include = FALSE} -dir_tree <- function(path, sub = ".", ...) { - withr::with_dir(path, fs::dir_tree(sub, ...)) -} - -lang_output <- function(x, lang) { - writeLines(c(sprintf("```%s", lang), x, "```")) -} -r_output <- function(x) lang_output(x, "r") -yaml_output <- function(x) lang_output(x, "yaml") -plain_output <- function(x) lang_output(x, "plain") -orderly_file <- function(...) { - system.file(..., package = "orderly2", mustWork = TRUE) -} +source("common.R") -knitr::opts_chunk$set( - collapse = TRUE) - path <- tempfile() fs::dir_create(path) path_git <- file.path(path, "git") @@ -43,7 +28,7 @@ sha <- gert::git_commit("initial", author = user, committer = user, if (gert::git_branch(repo = path_git) == "master") { gert::git_branch_move("master", "main", repo = path_git) } - + gert::git_clone(path_git, path_alice) gert::git_clone(path_git, path_bob) gert::git_clone(path_git, path_server) @@ -54,16 +39,6 @@ knitr::opts_hooks$set(as = function(options) { options$class.output <- options$as options }) - -.here <- getwd() -knitr::knit_hooks$set(inwd = function(before, options) { - if (before) { - setwd(options$inwd) - } else { - setwd(.here) - } - invisible() -}) ``` + +For the rest of the vignette we will evaluate commands from within this directory, by changing the directory to the path we've created: + +```r +setwd(path) +``` + # Creating your first orderly report An orderly report is a directory `src/` containing a file `orderly.R`. That file may have special commands in it, but for now we'll create one that is as simple as possible; we'll create some random data and save it to disk. This seems silly, but imagine this standing in for something like: @@ -93,8 +87,8 @@ r_output(readLines(file.path(path, "src/incoming_data/orderly.R"))) To run the report and create a new **packet**, use `orderly2::orderly_run()`: -```{r} -id <- orderly2::orderly_run("incoming_data", root = path) +```{r, inwd = path} +id <- orderly2::orderly_run("incoming_data") id ``` @@ -120,11 +114,11 @@ That's it! Notice that the initial script is just a plain R script, and you can Once created, you can then refer to this report by id and pull its files wherever you need them, both in the context of another orderly report or just to copy to your desktop to email someone. For example, to copy the file `data.rds` that we created to some location outside of orderly's control you could do -```{r} +```{r, inwd = path} dest <- tempfile() fs::dir_create(dest) orderly2::orderly_copy_files(id, files = c("final.rds" = "data.rds"), - dest = dest, root = path) + dest = dest) ``` which copies `data.rds` to some new temporary directory `dest` with name `final.rds`. This uses `orderly2`'s `outpack_` functions, which are designed to interact with outpack archives regardless of how they were created (`orderly2` is a program that creates `outpack` archives). Typically these are lower-level than `orderly_` functions. @@ -141,7 +135,7 @@ writeLines(c( 'd <- readRDS("incoming.rds")', 'png("analysis.png")', "plot(y ~ x, d)", - 'dev.off()'), + "dev.off()"), file.path(path, "src", "analysis", "orderly.R")) ``` @@ -162,8 +156,8 @@ Here, we've used `orderly2::orderly_dependency()` to pull in the file `data.rds` We can run this just as before, using `orderly2::orderly_run()`: -```{r} -id <- orderly2::orderly_run("analysis", root = path) +```{r, inwd = path} +id <- orderly2::orderly_run("analysis") ``` For more information on dependencies, see `vignette("dependencies")`. @@ -187,7 +181,7 @@ Let's add some additional annotations to the previous reports: ```{r, include = FALSE} code_data <- readLines(file.path(path, "src", "incoming_data", "orderly.R")) writeLines(c( - 'orderly2::orderly_strict_mode()', + "orderly2::orderly_strict_mode()", 'orderly2::orderly_resource("data.csv")', 'orderly2::orderly_artefact("Processed data", "data.rds")', "", @@ -201,8 +195,8 @@ r_output(readLines(file.path(path, "src/incoming_data/orderly.R"))) Here, we've added a block of special orderly commands; these could go anywhere, for example above the files that they refer to. If strict mode is enabled (see below) then `orderly2::orderly_resource` calls must go before the files are used as they will only be made available at that point (see below). -```{r} -id <- orderly2::orderly_run("incoming_data", root = path) +```{r, inwd = path} +id <- orderly2::orderly_run("incoming_data") ``` # Parameterised reports @@ -247,30 +241,28 @@ You can do anything in your report that switches on the value of a parameter: However, you should see parameters as relatively heavyweight things and try to have a consistent set over all packets created from a report. In this report we use it to control the size of the generated data set. -```{r} -id <- orderly2::orderly_run("random", list(n_samples = 15), root = path) +```{r, inwd = path} +id <- orderly2::orderly_run("random", list(n_samples = 15)) ``` Our resulting file has 15 rows, as the parameter we passed in affected the report: -```{r} +```{r, inwd = path} orderly2::orderly_copy_files(id, files = c("random.rds" = "data.rds"), - dest = dest, root = path) + dest = dest) readRDS(file.path(dest, "random.rds")) ``` You can use these parameters in orderly's search functions. For example we can find the most recent version of a packet by running: -```{r} -orderly2::orderly_search('latest(name == "random")', - root = path) +```{r, inwd = path} +orderly2::orderly_search('latest(name == "random")') ``` But we can also pass in parameter queries here: -```{r} -orderly2::orderly_search('latest(name == "random" && parameter:n_samples > 10)', - root = path) +```{r, inwd = path} +orderly2::orderly_search('latest(name == "random" && parameter:n_samples > 10)') ``` These can be used within `orderly2::orderly_dependency()` (the `name == "random"` part is implied by the first `name` argument), for example @@ -317,15 +309,15 @@ writeLines(c( 'd <- read.csv("data.csv")', 'png("analysis.png")', "plot(y ~ x, d)", - 'dev.off()'), + "dev.off()"), file.path(path, "src", "use_shared", "orderly.R")) r_output(readLines(file.path(path, "src/use_shared/orderly.R"))) ``` We can run this: -```{r} -id <- orderly2::orderly_run("use_shared", root = path) +```{r, inwd = path} +id <- orderly2::orderly_run("use_shared") ``` In the resulting archive, the file that was used from the shared directory is present: @@ -364,24 +356,24 @@ withr::with_dir(file.path(path, "src/incoming_data"), sys.source("orderly.R", new.env(parent = .GlobalEnv))) ``` -```{r} -orderly2::orderly_cleanup_status("incoming_data", root = path) +```{r, inwd = path} +orderly2::orderly_cleanup_status("incoming_data") ``` If you have files here that are unknown to orderly it will tell you about them and prompt you to tell it about them explicitly. You can clean up generated files by running (as suggested in the message): -```{r} -orderly2::orderly_cleanup("incoming_data", root = path) +```{r, inwd = path} +orderly2::orderly_cleanup("incoming_data") ``` There is a `dry_run = TRUE` argument you can pass if you want to see what would be deleted without using the status function. You can also keep these files out of git by using the `orderly2::orderly_gitignore_update` function: -```{r} -orderly2::orderly_gitignore_update("incoming_data", root = path) +```{r, inwd = path} +orderly2::orderly_gitignore_update("incoming_data") ``` This creates (or updates) a `.gitignore` file within the report so that generated files will not be included by git. If you have already accidentally committed them then the gitignore has no real effect and you should do some git surgery, see the git manuals or this [handy, if profane, guide](https://ohshitgit.com/). @@ -392,23 +384,21 @@ If you delete packets from your `archive/` directory then this puts `orderly2` i At the moment, we have two copies of the `incoming_data` task: -```{r} +```{r, inwd = path} orderly2::orderly_metadata_extract( name = "incoming_data", - extract = c(time = "time.start"), - root = path) + extract = c(time = "time.start")) ``` -```{r include = FALSE} -id_latest <- orderly2::orderly_search("latest", name = "incoming_data", - root = path) +```{r include = FALSE, inwd = path} +id_latest <- orderly2::orderly_search("latest", name = "incoming_data") unlink(file.path(path, "archive", "incoming_data", id_latest), recursive = TRUE) ``` When we run the `analysis` task, it will pull in the most recent version (`r inline(id_latest)`). However, if you had deleted this manually (e.g., to save space or accidentally) or corrupted it (e.g., by opening some output in Excel and letting it save changes) it will not be able to be included, and running `analysis` will fail: -```{r error = TRUE} -orderly2::orderly_run("analysis", root = path) +```{r error = TRUE, inwd = path} +orderly2::orderly_run("analysis") ``` The error here tries to be fairly informative, telling us that we failed because when copying files from `r inline(id_latest)` we found that the packet was corrupt, because the file `data.rds` was not found in the archive. It also suggests a fix; we can tell `orderly2` that `r inline(id_latest)` is "orphaned" and should not be considered for inclusion when we look for dependencies. @@ -423,16 +413,16 @@ r_output( or we can validate *all* the packets we have: -```{r} -orderly2::orderly_validate_archive(action = "orphan", root = path) +```{r, inwd = path} +orderly2::orderly_validate_archive(action = "orphan") ``` If we had the option `core.require_complete_tree` enabled, then this process would also look for any packets that used our now-deleted packet and orphan those too, as we no longer have a complete tree that includes them. If you want to remove references to the orphaned packets, you can use `orderly2::orderly_prune_orphans()` to remove them entirely: -```{r} -orderly2::orderly_prune_orphans(root = path) +```{r, inwd = path} +orderly2::orderly_prune_orphans() ``` # Debugging and coping with errors @@ -476,8 +466,8 @@ As can be perhaps inferred from the filenames, the files `.outpack/metadata/ %\VignetteEncoding{UTF-8} --- -``` {r include = FALSE} -lang_output <- function(x, lang) { - writeLines(c(sprintf("```%s", lang), x, "```")) -} -r_output <- function(x) lang_output(x, "r") -json_output <- function(x) lang_output(x, "json") -yaml_output <- function(x) lang_output(x, "yaml") -plain_output <- function(x) lang_output(x, "plain") -knitr::opts_chunk$set( - collapse = TRUE, - error = FALSE) +```{r, include = FALSE} +source("common.R") ``` **You may not need to read this: the intended readers are authors of `orderly2` plugins, not users of such plugins.** @@ -127,7 +118,8 @@ writeLines(c( "License: CC0", "Title: Orderly Database Example Plugin", "Description: Simple example of an orderly plugin.", - "Authors@R: person('Orderly Authors', role = c('aut', 'cre'), email = 'email@example.com')", + "Authors@R: person('Orderly Authors', role = c('aut', 'cre'),", + " email = 'email@example.com')", "Imports: orderly2"), file.path(path_pkg, "DESCRIPTION")) @@ -219,7 +211,7 @@ plain_output(readLines(file.path(path_pkg, "R/plugin.R"))) ### Trying it out -In order to test your package, it needs to be loaded. You can do this by either installing the package or by using `pkgload::load_all()` (you may find doing so with `pkgload::load_all(export_all = FALSE)` gives the most reliably experience. +In order to test your package, it needs to be loaded. You can do this by either installing the package or by using `pkgload::load_all()` (you may find doing so with `pkgload::load_all(export_all = FALSE)` gives the most reliable experience. ```{r, eval = FALSE} pkgload::load_all() @@ -305,30 +297,30 @@ Taking this a step further, we can also specify a [schema](https://json-schema.o dir.create(file.path(path_pkg, "inst"), FALSE, TRUE) path_schema <- file.path(path_pkg, "inst", "schema.json") writeLines(c( - '{', + "{", ' "$schema": "http://json-schema.org/draft-07/schema#",', - '', + "", ' "type": "object",', ' "additionalProperties": {', ' "type": "object",', ' "properties": {', ' "query": {', ' "type": "string"', - ' },', + " },", ' "rows": {', ' "type": "number"', - ' },', + " },", ' "cols": {', ' "type": "array",', ' "items": {', ' "type": "character"', - ' }', - ' }', - ' },', + " }", + " }", + " },", ' "required": ["query", "rows", "cols"],', ' "additionalProperties": false', - ' }', - '}'), + " }", + "}"), path_schema) json_output(readLines(path_schema)) ``` diff --git a/vignettes/query.Rmd b/vignettes/query.Rmd index 765ae7c3..4f28328c 100644 --- a/vignettes/query.Rmd +++ b/vignettes/query.Rmd @@ -186,8 +186,7 @@ We can also search up the tree and then down to find `A` from `D` e.g. ``` orderly_query( quote(usedby(single(uses(name == "D")))), - name = "A", - root = root) + name = "A") ``` note as `E` is the only packet which uses `D` we do not need to add a `name == "E"` clause. @@ -197,8 +196,7 @@ We can combine `usedby` and `uses` in more complex searches, such as to find `D` ``` orderly_query( quote(usedby(single(uses(single(usedby(latest(name == "C")) && name == "A"))) && name == "E"))), - name = "D", - root = root) + name = "D") ``` ## Possible future queries and interface improvements diff --git a/vignettes/troubleshooting.Rmd b/vignettes/troubleshooting.Rmd index ba412748..efe21e02 100644 --- a/vignettes/troubleshooting.Rmd +++ b/vignettes/troubleshooting.Rmd @@ -24,10 +24,20 @@ gert::git_init(path) gert::git_add(".", repo = path) user <- "author " gert::git_commit("initial", author = user, committer = user, repo = path) + +.here <- getwd() +knitr::knit_hooks$set(inwd = function(before, options) { + if (before) { + setwd(options$inwd) + } else { + setwd(.here) + } + invisible() +}) ``` -```{r, error = TRUE} -orderly2::orderly_run("data", root = path) +```{r, error = TRUE, inwd = path} +orderly2::orderly_run("data") ``` which may have directed you to this very page. If you just want to continue working anyway, then run the suggested command: @@ -38,14 +48,14 @@ options(orderly_git_error_is_warning = TRUE) after which things will work with a warning the first time that session: -```{r} -orderly2::orderly_run("data", root = path) +```{r, inwd = path} +orderly2::orderly_run("data") ``` subsequent calls will not display the warning: -```{r} -orderly2::orderly_run("data", root = path) +```{r, inwd = path} +orderly2::orderly_run("data") ``` ```{r, include = FALSE}