Skip to content

Commit

Permalink
add fields as query default in api call, same with limit, offset; now…
Browse files Browse the repository at this point in the history 
… file detail directly use fields = _all, so only one request issued. Then also update a uploading logic for folder/multiple files
  • Loading branch information
@tengfei
tengfei committed Jan 10, 2017
1 parent 5b14c74 commit 54488bc
Show file tree
Hide file tree
Showing 5 changed files with 113 additions and 27 deletions.
9 changes: 8 additions & 1 deletion R/api-http.R
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,12 @@
#' a length 1 vector as a vector, wrap in \code{I()}.
#' @param limit How many results to return
#' @param offset The point at which to start displaying them
#' @param fields All API calls take the optional query parameter fields.
#' This parameter enables you to specify the fields you want to be returned
#' when listing resources (e.g. all your projects) or getting details of a
#' specific resource (e.g. a given project). For example, fields="id,name,size"
#' to return the fields id, name and size for files. More details please check
#' \url{http://docs.sevenbridges.com/reference#section-general-api-information}
#' @param base_url defeault is \code{"https://api.sbgenomics.com/v2"}
#' @param ... passed to GET/POST/PUT/DELETE/PATCH call.
#'
Expand All @@ -47,6 +53,7 @@ api = function(token = NULL, version = 'v2', path = NULL,
encode = c("json", "form", "multipart"),
limit = getOption("sevenbridges")$limit,
offset = getOption("sevenbridges")$offset,
fields = NULL,
base_url = paste0("https://api.sbgenomics.com/", version, "/"),
...) {

Expand All @@ -63,7 +70,7 @@ api = function(token = NULL, version = 'v2', path = NULL,
)

# setup query
query = c(query, list(limit = limit, offset = offset))
query = c(query, list(limit = limit, offset = offset, fields = fields))
idx = !sapply(query, is.null)
if (any(idx)) {
query <- query[idx]
Expand Down
29 changes: 14 additions & 15 deletions R/class-auth.R
View file
Original file line number Diff line number Diff line change
Expand Up @@ -464,13 +464,15 @@ Auth <- setRefClass("Auth", fields = list(from = "character",
api = function(...,
limit = getOption("sevenbridges")$"limit",
offset = getOption("sevenbridges")$"offset",
fields = NULL,
complete = FALSE) {

'
This call returns all API paths, and pass arguments
to api() function and input token and url automatically'

req = sevenbridges::api(token, base_url = url, limit = limit, offset = offset, ...)
req = sevenbridges::api(token, base_url = url, limit = limit,
offset = offset, fields = fields, ...)
req = status_check(req)

if (complete) {
Expand Down Expand Up @@ -579,7 +581,9 @@ Auth <- setRefClass("Auth", fields = list(from = "character",
}

# build query

.query = list(project = project)

if (length(metadata)) {
new.meta = unlist(metadata)
names(new.meta) = sapply(names(new.meta),
Expand All @@ -591,6 +595,12 @@ Auth <- setRefClass("Auth", fields = list(from = "character",
.query = c(.query, list(origin.task = origin.task))
}

if(detail){
.query = c(.query, list(fields = "_all"))
}



.split_item = function(x, list_name = NULL) {
if (length(x) > 1) {
names(x) = rep(list_name, length(x))
Expand All @@ -612,6 +622,8 @@ Auth <- setRefClass("Auth", fields = list(from = "character",
.query = c(.query, .new_tag)
}



if (is.null(name)) {
# if no id, no name, list all

Expand Down Expand Up @@ -658,20 +670,7 @@ Auth <- setRefClass("Auth", fields = list(from = "character",
res = m.match(res, id = id, name = name, exact = exact)
})

if (length(res)) {
if (detail) {
if (is(res, "FilesList")) {
ids = sapply(res, function(x){ x$id })
} else {
ids = res$id
}
lst = lapply(ids, function(id) {
req = api(path = paste0("files/", id), method = "GET", ...)
.asFiles(req)
})
res = FilesList(lst)
}
} else {
if (!length(res)) {
return(NULL)
}
res = setAuth(res, .self, "Files")
Expand Down
43 changes: 34 additions & 9 deletions R/class-project.R
View file
Original file line number Diff line number Diff line change
Expand Up @@ -280,7 +280,7 @@ Project <- setRefClass("Project", contains = "Item",
manifest_file = NULL,
manifest_metadata = TRUE,
subset, select,
verbal = FALSE,
verbal = NULL,
...) {

# upload via a manifest
Expand Down Expand Up @@ -353,10 +353,13 @@ Project <- setRefClass("Project", contains = "Item",

}

if(is.null(verbal))
verbal <- FALSE

# if verbal = TRUE, print file uploading progress info for each file
# if verbal = FALSE, print all files uploading progress in single bar
if(!verbal){
message("uploaded files progress:")
message("files uploading progress:")
pb <- txtProgressBar(min = 0, max = nrow(manf.sub), style = 3)
}

Expand Down Expand Up @@ -390,15 +393,35 @@ Project <- setRefClass("Project", contains = "Item",

# if filename is a list
if (length(filename) > 1) {
for (fl in filename) {
message(fl)
if (file.info(fl)$size > 0) {
upload(fl, metadata = metadata,
overwrite = overwrite, ...)
if(is.null(verbal))
verbal <- FALSE
# if verbal = TRUE, print file uploading progress info for each file
# if verbal = FALSE, print all files uploading progress in single bar
if(!verbal){
message("files uploading progress:")
pb <- txtProgressBar(min = 0, max = length(filename), style = 3)
}
for (i in 1:length(filename)) {
fl = filename[i]
if(verbal){
message(fl)
if (file.info(fl)$size > 0) {
upload(fl, metadata = metadata,
overwrite = overwrite, verbal = verbal, ...)
} else {
warning("skip uploading: empty file")
}
} else {
warning("skip uploading: empty file")
if (file.info(fl)$size > 0) {
upload(fl, metadata = metadata,
overwrite = overwrite, verbal = verbal, ...)
setTxtProgressBar(pb, i)
}
}
}
if(!verbal){
close(pb)
}
return(invisible())
}

Expand All @@ -407,7 +430,7 @@ Project <- setRefClass("Project", contains = "Item",
message("Upload all files in the folder: ", filename)
fls = list.files(filename, recursive = TRUE, full.names = TRUE)
upload(fls, metadata = metadata,
overwrite = overwrite, ...)
overwrite = overwrite, verbal = verbal, ...)
return(invisible())
}

Expand All @@ -420,6 +443,8 @@ Project <- setRefClass("Project", contains = "Item",
project_id = id,
metadata = metadata, ...)

if(is.null(verbal))
verbal <- TRUE
u$upload_file(metadata = metadata,
overwrite = overwrite,
verbal = verbal)
Expand Down
8 changes: 6 additions & 2 deletions R/class-upload.R
View file
Original file line number Diff line number Diff line change
Expand Up @@ -181,7 +181,6 @@ Upload <- setRefClass("Upload", contains = "Item",
# size <<- res$size
parallel_uploads <<- as.logical(res$parallel_uploads)
part_length <<- as.integer(ceiling(.self$size/part_size))
message("Initialized")
invisible(res)

},
Expand Down Expand Up @@ -216,16 +215,19 @@ Upload <- setRefClass("Upload", contains = "Item",
res
},

upload_file = function(metadata = list(), overwrite = FALSE, verbal = TRUE) {
upload_file = function(metadata = list(), overwrite = FALSE,
verbal = TRUE) {

# make this one easy to use

res <- upload_init(overwrite = overwrite)
N <- part_length
if(verbal){
message("size: ", size)
message("part_size: ", part_size)
message("part_length: ", part_length)
message("parallel_uploads: ", parallel_uploads)
}

if(verbal){
pb <- txtProgressBar(min = 0, max = N, style = 3)
Expand Down Expand Up @@ -255,12 +257,14 @@ Upload <- setRefClass("Upload", contains = "Item",
close(con)
.end = Sys.time()
.diff = .end - .start
if(verbal){
message("file uploading complete in: ",
ceiling(as.numeric(.diff)), " ", attr(.diff, "unit") )

message("Estimated uploading speed: ",
ceiling(size/1024/1024/as.numeric(.diff)),
" Mb/", attr(.diff, "unit"))
}

# # when we complete we could add meta
# meta <- .self$metadata$asList()
Expand Down
51 changes: 51 additions & 0 deletions vignettes/api.Rmd
View file
Original file line number Diff line number Diff line change
Expand Up @@ -847,6 +847,57 @@ a$app(visibility = "public", name = "bed converter", complete = TRUE)

<div align="right"><a href="#top">top</a></div>

## Ouery parameter 'fields'

Please read the documentation [detail](http://docs.sevenbridges.com/reference#section-general-api-information)

All API calls take the optional query parameter fields. This parameter enables you
to specify the fields you want to be returned when listing resources
(e.g. all your projects) or getting details of a specific resource (e.g. a given project).

The fields parameter can be used in the following ways:

1. No fields parameter specified: calls return default fields. For calls that return
complete details of a single resource, this is all their properties; for calls that list
resources of a certain type, this is some default properties.

2. The fields parameter can be set to a list of fields: for example, to return the fields
id, name and size for files in a project, you may issue the call `p$file(fields = "id,name,size")`

3. The fields parameter can be used to exclude a specific file: if you wish to omit certain
field from the response, do so using the fields parameter with the prefix !: for example,
to get the details of a file without listing its metadata, issue a call
`p$file(fields = '!metadata')` The entire metadata field will be removed from the response.

4. The fields parameter can be used to include or omit certain nested fields, in the same
way as listed in 2 and 3 above: for example, you can use metadata.sample_id or
origin.task for files.

5. To see all fields for a resource, specify `fields="_all"`. This returns all fields
for each resource returned. Note that if you are getting the details of a specific resource,
the use of `fields="_all"` won't return any more properties than would have been shown
without this parameter – the use case is instead for when you are listing details
of many resources. Please use with care if your resource has particularly large fields;
for example, the raw field for an app resource contains the complete CWL specification
of the app which can result in bulky response if listing many apps.

6. Negations and nesting can be combined freely, so, for example, you can issue `p$file(fields="id,name,status,!metadata.library,!origin")` or `p$task(fields="!inputs,!outputs")`.

Please try following examples

```{r}
## default fields id, name, project
p$file()
## return file(s) with id, name, siae information
p$file(fields = "id,name,size")
## return file(s) with all available info
p$file(detail = TRUE)
## same as above
p$file(fields = "_all")
```

<div align="right"><a href="#top">top</a></div>

## Rate Limits

This call returns information about your current rate limit.
Expand Down

0 comments on commit 54488bc

Please sign in to comment.