From 7e5d5d13a14321d0c300739874d8dee25449668a Mon Sep 17 00:00:00 2001 From: Romain Franconville Date: Wed, 22 Jan 2020 16:27:15 -0500 Subject: [PATCH 1/2] Switch neuprint_find_neurons to a custom query, doing all the roiInfo extracting on the server side. Speeds up the function (same results are returned). --- R/roi.R | 39 +++++++++++++++++++-------------------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/R/roi.R b/R/roi.R index a40da0ba..240a10ef 100644 --- a/R/roi.R +++ b/R/roi.R @@ -29,27 +29,26 @@ neuprint_find_neurons <- function(input_ROIs, stop("Invalid stauses provided. Statuses must be NULL to accept any body status, or on of: ", possible.statuses) } } - all_segments = ifelse(all_segments,"true","false") + conn <- neuprint_login(conn) + all_segments.json <- ifelse(all_segments,"Segment","Neuron") + dp <- neuprint_dataset_prefix(dataset, conn=conn) + prefixed <- paste0(dp, all_segments.json) + #all_segments = ifelse(all_segments,"true","false") roicheck = neuprint_check_roi(rois=unique(c(input_ROIs,output_ROIs)), dataset = dataset, conn = conn, ...) - Payload = noquote(sprintf('{"dataset":"%s","input_ROIs":%s,"output_ROIs":%s,"statuses":%s,"all_segments":%s}', - dataset, jsonlite::toJSON(input_ROIs), - jsonlite::toJSON(output_ROIs), - ifelse(is.null(statuses),jsonlite::toJSON(list()),jsonlite::toJSON(statuses)), - all_segments)) - class(Payload) = "json" - found.neurons = neuprint_fetch(path = 'api/npexplorer/findneurons', body = Payload, conn = conn, ...) - columns = unlist(found.neurons[[1]]) - keep = !columns%in%c("roiInfo","rois") - neurons = data.frame() - extract = lapply(found.neurons[[2]], function(f) nullToNA(t(as.matrix(f[keep])))) - neurons = as.data.frame(do.call(rbind, extract)) - colnames(neurons) = columns[keep] - rownames(neurons) = neurons$bodyid - innervation = lapply(found.neurons[[2]], function(f) - extract_connectivity_df(rois = c(input_ROIs,output_ROIs), - json=unlist(f[columns=="roiInfo"]))) - innervation = do.call(rbind,innervation) - cbind(neurons,innervation) + + cypher <- sprintf(paste("MATCH (neuron :%s)", + "WHERE %s %s", + "WITH neuron AS neuron, apoc.convert.fromJsonMap(neuron.roiInfo) AS roiInfo", + "RETURN neuron.bodyId AS bodyid, neuron.instance AS name, neuron.type AS type, neuron.pre AS pre, neuron.post AS post,%s %s"), + prefixed, + ifelse(is.null(statuses),"",paste0("(",paste("neuron.status = ",statuses,collapse=" OR "),") AND")), + paste0("(",paste0("neuron.",c(input_ROIs,output_ROIs),collapse=" AND "),")"), + ifelse(is.null(input_ROIs),"",paste0("roiInfo.`",input_ROIs,"`.post AS `",input_ROIs,".post`",collapse=",")), + ifelse(is.null(output_ROIs),"",paste0(",roiInfo.`",output_ROIs,"`.pre AS `",output_ROIs,".pre`")) + ) + nc <- neuprint_fetch_custom(cypher=cypher, conn = conn, ...) + results <- na.omit(neuprint_list2df(nc)) ## NAs mean one of our conditions is not met + results } #' @export From 0658bd7f64b2fb1c6343c10fbcaf0f38606aadc4 Mon Sep 17 00:00:00 2001 From: Romain Franconville Date: Thu, 23 Jan 2020 09:02:21 -0500 Subject: [PATCH 2/2] Drop prefixed Neurons/Segments --- R/connectivity.R | 30 +++++++++++++++--------------- R/name.R | 18 +++++++++--------- R/roi.R | 10 ++++------ 3 files changed, 28 insertions(+), 30 deletions(-) diff --git a/R/connectivity.R b/R/connectivity.R index f903e902..d4e57d90 100644 --- a/R/connectivity.R +++ b/R/connectivity.R @@ -9,7 +9,7 @@ neuprint_get_adjacency_matrix <- function(bodyids, dataset = NULL, all_segments = TRUE, conn = NULL, ...){ dataset <- check_dataset(dataset) conn=neuprint_login(conn) - dp=neuprint_dataset_prefix(dataset, conn=conn) + #dp=neuprint_dataset_prefix(dataset, conn=conn) all_segments.json = ifelse(all_segments,"Segment","Neuron") namefield=neuprint_name_field(conn) cypher = sprintf( @@ -21,7 +21,7 @@ neuprint_get_adjacency_matrix <- function(bodyids, dataset = NULL, all_segments jsonlite::toJSON(as.numeric(unique(unlist( bodyids )))), - paste0(dataset, "_", all_segments.json), + all_segments.json, namefield, namefield ) @@ -71,7 +71,7 @@ neuprint_connection_table <- function(bodyids, prepost = c("PRE","POST"), roi = } dp=neuprint_dataset_prefix(dataset, conn=conn) - prefixed=paste0(dp, all_segments.json) + #prefixed=paste0(dp, all_segments.json) cypher = sprintf(paste("WITH %s AS bodyIds UNWIND bodyIds AS bodyId", "MATCH (a:`%s`)-[c:ConnectsTo]->(b:`%s`)", "WHERE %s.bodyId=bodyId", @@ -79,8 +79,8 @@ neuprint_connection_table <- function(bodyids, prepost = c("PRE","POST"), roi = "RETURN a.bodyId AS %s, b.bodyId AS %s, k AS roi,", "apoc.convert.fromJsonMap(c.roiInfo)[k].post AS weight"), jsonlite::toJSON(unique(as.numeric(unlist(bodyids)))), - prefixed, - prefixed, + all_segments.json, + all_segments.json, ifelse(prepost=="POST","a","b"), ifelse(is.null(roi),"keys(apoc.convert.fromJsonMap(c.roiInfo))",paste("['",paste(roi,collapse="','"),"']",sep="")), ifelse(prepost=="POST","bodyid","partner"), @@ -179,11 +179,11 @@ neuprint_simple_connectivity <- function(bodyids, dataset <- check_dataset(dataset) find_inputs = ifelse(prepost=="PRE", "false","true") if(length(bodyids)>10){ - m = do.call(rbind, pbapply::pblapply(bodyids, function(bi) tryCatch(neuprint_simple_connectivity( + m = Reduce(function(x,y,...) dplyr::full_join(x,y,by=c("name",ifelse(prepost=="PRE","output","input"),"type")),(pbapply::pblapply(bodyids, function(bi) tryCatch(neuprint_simple_connectivity( bodyids = bi, prepost = prepost, dataset = dataset, conn = conn, ...), - error = function(e) NULL))) + error = function(e) NULL)))) return(m) } Payload = noquote(sprintf('{"dataset":"%s","neuron_ids":%s,"find_inputs":%s}', @@ -244,8 +244,8 @@ neuprint_get_paths <- function(body_pre,body_post,n,weightT=5,roi=NULL,dataset = } all_segments.json <- ifelse(all_segments,"Segment","Neuron") - dp <- neuprint_dataset_prefix(dataset, conn=conn) - prefixed <- paste0(dp, all_segments.json) + #dp <- neuprint_dataset_prefix(dataset, conn=conn) + #prefixed <- paste0(dp, all_segments.json) cypher <- sprintf(paste("WITH [%s,%s] AS bodies", "UNWIND bodies[0] AS bodypre", @@ -257,10 +257,10 @@ neuprint_get_paths <- function(body_pre,body_post,n,weightT=5,roi=NULL,dataset = ), jsonlite::toJSON(unique(as.numeric(unlist(body_pre)))), jsonlite::toJSON(unique(as.numeric(unlist(body_post)))), - prefixed, + all_segments.json, n[1]-1, n[2], - prefixed, + all_segments.json, weightT, ifelse(is.null(roi),"",roi) ) @@ -306,8 +306,8 @@ neuprint_get_shortest_paths <- function(body_pre,body_post,weightT=5,roi=NULL,da dataset <- check_dataset(dataset) conn <- neuprint_login(conn) all_segments.json <- ifelse(all_segments,"Segment","Neuron") - dp <- neuprint_dataset_prefix(dataset, conn=conn) - prefixed <- paste0(dp, all_segments.json) + #dp <- neuprint_dataset_prefix(dataset, conn=conn) + #prefixed <- paste0(dp, all_segments.json) if(!is.null(roi)){ roicheck = neuprint_check_roi(rois=roi, dataset = dataset, conn = conn, ...) @@ -325,8 +325,8 @@ neuprint_get_shortest_paths <- function(body_pre,body_post,weightT=5,roi=NULL,da ), jsonlite::toJSON(unique(as.numeric(unlist(body_pre)))), jsonlite::toJSON(unique(as.numeric(unlist(body_post)))), - prefixed, - prefixed, + all_segments.json, + all_segments.json, weightT, ifelse(is.null(roi),"",roi) ) diff --git a/R/name.R b/R/name.R index 61c5930c..760f7301 100644 --- a/R/name.R +++ b/R/name.R @@ -12,12 +12,12 @@ neuprint_get_neuron_names <- function(bodyids, dataset = NULL, all_segments = TR # Get a default dataset if none specified dataset <- check_dataset(dataset) conn=neuprint_login(conn) - dp=neuprint_dataset_prefix(dataset, conn=conn) + #dp=neuprint_dataset_prefix(dataset, conn=conn) - all_segments = ifelse(all_segments,"Segment","Neuron") + all_segments.json = ifelse(all_segments,"Segment","Neuron") cypher = sprintf("WITH %s AS bodyIds UNWIND bodyIds AS bodyId MATCH (n:`%s`) WHERE n.bodyId=bodyId RETURN n.instance AS name", jsonlite::toJSON(as.numeric(unlist(bodyids))), - paste0(dp,all_segments)) + all_segments.json) nc = neuprint_fetch_custom(cypher=cypher, conn = conn, ...) d = unlist(lapply(nc$data,nullToNA)) names(d) = bodyids @@ -38,7 +38,7 @@ neuprint_get_meta <- function(bodyids, dataset = NULL, all_segments = TRUE, conn dataset <- check_dataset(dataset) all_segments = ifelse(all_segments,"Segment","Neuron") conn=neuprint_login(conn) - dp=neuprint_dataset_prefix(dataset, conn=conn) + #dp=neuprint_dataset_prefix(dataset, conn=conn) cypher = sprintf( paste( "WITH %s AS bodyIds UNWIND bodyIds AS bodyId ", @@ -46,7 +46,7 @@ neuprint_get_meta <- function(bodyids, dataset = NULL, all_segments = TRUE, conn "RETURN n.bodyId AS bodyid, n.%s AS name, n.type AS type, n.status AS status, n.size AS voxels, n.pre AS pre, n.post AS post, n.cropped AS cropped" ), jsonlite::toJSON(as.numeric(unlist(bodyids))), - paste0(dp, all_segments), + all_segments, neuprint_name_field(conn) ) nc = neuprint_fetch_custom(cypher=cypher, conn = conn, include_headers = F, ...) @@ -66,12 +66,12 @@ neuprint_get_meta <- function(bodyids, dataset = NULL, all_segments = TRUE, conn neuprint_get_roiInfo <- function(bodyids, dataset = NULL, all_segments = TRUE, conn = NULL, ...){ dataset <- check_dataset(dataset) conn=neuprint_login(conn) - dp=neuprint_dataset_prefix(dataset, conn=conn) + #dp=neuprint_dataset_prefix(dataset, conn=conn) all_segments = ifelse(all_segments,"Segment","Neuron") cypher = sprintf( "WITH %s AS bodyIds UNWIND bodyIds AS bodyId MATCH (n:`%s`) WHERE n.bodyId=bodyId RETURN n.bodyId AS bodyid, n.roiInfo AS roiInfo", jsonlite::toJSON(as.numeric(unlist(bodyids))), - paste0(dp, all_segments) + all_segments ) nc = neuprint_fetch_custom(cypher=cypher, conn = conn, ...) lc <- lapply(nc$data,function(x){cbind(bodyid=x[[1]],as.data.frame(t(unlist(jsonlite::fromJSON(x[[2]])))))}) @@ -99,10 +99,10 @@ neuprint_get_roiInfo <- function(bodyids, dataset = NULL, all_segments = TRUE, c neuprint_search <- function(search, meta = TRUE, all_segments = TRUE, dataset = NULL, conn = NULL, ...){ dataset <- check_dataset(dataset) conn=neuprint_login(conn) - dp=neuprint_dataset_prefix(dataset, conn=conn) + # dp=neuprint_dataset_prefix(dataset, conn=conn) all_segments.cypher = ifelse(all_segments,"Segment","Neuron") cypher = sprintf("MATCH (n:`%s`) WHERE n.%s=~'%s' RETURN n.bodyId", - paste0(dp, all_segments.cypher), + all_segments.cypher, neuprint_name_field(conn), search) nc = neuprint_fetch_custom(cypher=cypher, ...) diff --git a/R/roi.R b/R/roi.R index 240a10ef..e756a81d 100644 --- a/R/roi.R +++ b/R/roi.R @@ -31,16 +31,14 @@ neuprint_find_neurons <- function(input_ROIs, } conn <- neuprint_login(conn) all_segments.json <- ifelse(all_segments,"Segment","Neuron") - dp <- neuprint_dataset_prefix(dataset, conn=conn) - prefixed <- paste0(dp, all_segments.json) - #all_segments = ifelse(all_segments,"true","false") + roicheck = neuprint_check_roi(rois=unique(c(input_ROIs,output_ROIs)), dataset = dataset, conn = conn, ...) cypher <- sprintf(paste("MATCH (neuron :%s)", "WHERE %s %s", "WITH neuron AS neuron, apoc.convert.fromJsonMap(neuron.roiInfo) AS roiInfo", "RETURN neuron.bodyId AS bodyid, neuron.instance AS name, neuron.type AS type, neuron.pre AS pre, neuron.post AS post,%s %s"), - prefixed, + all_segments.json, ifelse(is.null(statuses),"",paste0("(",paste("neuron.status = ",statuses,collapse=" OR "),") AND")), paste0("(",paste0("neuron.",c(input_ROIs,output_ROIs),collapse=" AND "),")"), ifelse(is.null(input_ROIs),"",paste0("roiInfo.`",input_ROIs,"`.post AS `",input_ROIs,".post`",collapse=",")), @@ -56,11 +54,11 @@ neuprint_find_neurons <- function(input_ROIs, neuprint_bodies_in_ROI <- function(roi, dataset = NULL, all_segments = TRUE, conn = NULL, ...){ dataset <- check_dataset(dataset) conn=neuprint_login(conn) - dp=neuprint_dataset_prefix(dataset, conn=conn) + #dp=neuprint_dataset_prefix(dataset, conn=conn) all_segments = ifelse(all_segments,"Segment","Neuron") roicheck = neuprint_check_roi(rois=roi, dataset = dataset, conn = conn, ...) cypher = sprintf("MATCH (n :`%s`) WHERE n.%s WITH n AS n, apoc.convert.fromJsonMap(n.roiInfo) AS roiInfo RETURN n.bodyId AS bodyid, n.size AS voxels, n.pre, n.post, roiInfo.%s.pre, roiInfo.%s.post", - paste0(dp, all_segments), + all_segments, roi, roi, roi)