From 90a6746931aa5bd16cfa57b973599afa25ab49fd Mon Sep 17 00:00:00 2001
From: Zane Billings <39701545+wz-billings@users.noreply.github.com>
Date: Tue, 6 Jul 2021 15:45:49 -0400
Subject: [PATCH 01/16] feat: adds code for computing CI to roc.test delong
 method.

---
 R/roc.test.R | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/R/roc.test.R b/R/roc.test.R
index 30550bf..8e8cdc2 100644
--- a/R/roc.test.R
+++ b/R/roc.test.R
@@ -305,9 +305,12 @@ roc.test.roc <- function(roc1, roc2,
 	if (method == "delong") {
 		if (paired) {
 			stat <- delong.paired.test(roc1, roc2)
+			stat.ci <- ci.delong.paired(roc1, roc2)
 			names(stat) <- "Z"
 			htest$statistic <- stat
 			htest$method <- "DeLong's test for two correlated ROC curves"
+			htest$ci.upper <- stat.ci$upper
+			htest$ci.lower <- stat.ci$lower
 			
 			if (alternative == "two.sided")
 				pval <- 2*pnorm(-abs(stat))

From 0c86a0ee9428a81622afbddc95bfe63ce2692dde Mon Sep 17 00:00:00 2001
From: Zane Billings <39701545+wz-billings@users.noreply.github.com>
Date: Tue, 6 Jul 2021 16:48:27 -0400
Subject: [PATCH 02/16] feat: computes delong paired test CI

---
 R/delong.R | 49 +++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 49 insertions(+)

diff --git a/R/delong.R b/R/delong.R
index 20ab494..16c0e08 100644
--- a/R/delong.R
+++ b/R/delong.R
@@ -111,6 +111,55 @@ ci.auc.delong <- function(roc, conf.level) {
   return(ci)
 }
 
+# function to calculate the CI
+ci.delong.paired <- function(roc1, roc2, ci.level = NULL) {
+	
+	# check whether CI level was specified correctly or not.
+	if (is.null(ci.level)) {
+		ci.level <- 0.95
+		message("ci.level not specified, defaults to 0.95.")
+	} else if (0 > ci.level | 1 < ci.level) {
+		ci.level <- 0.95
+		message("ci.level must be between 0 and 1. defaulting to 0.95.")
+	} else if (!is.numeric(ci.level)) {
+		ci.level <- 0.95
+		message("ci.level must be numeric between 0 and 1. defaulting to 0.95.")
+	}
+	
+	# Code from delong test for intermediate calculations
+	n <- length(roc1$controls)
+	m <- length(roc1$cases)
+	
+	VR <- delongPlacements(roc1)
+	VS <- delongPlacements(roc2)
+	
+	SX <- matrix(NA, ncol=2, nrow=2)
+	SX[1,1] <- sum((VR$X - VR$theta) * (VR$X - VR$theta))/(m-1)
+	SX[1,2] <- sum((VR$X - VR$theta) * (VS$X - VS$theta))/(m-1)
+	SX[2,1] <- sum((VS$X - VS$theta) * (VR$X - VR$theta))/(m-1)
+	SX[2,2] <- sum((VS$X - VS$theta) * (VS$X - VS$theta))/(m-1)
+	
+	SY <- matrix(NA, ncol=2, nrow=2)
+	SY[1,1] <- sum((VR$Y - VR$theta) * (VR$Y - VR$theta))/(n-1)
+	SY[1,2] <- sum((VR$Y - VR$theta) * (VS$Y - VS$theta))/(n-1)
+	SY[2,1] <- sum((VS$Y - VS$theta) * (VR$Y - VR$theta))/(n-1)
+	SY[2,2] <- sum((VS$Y - VS$theta) * (VS$Y - VS$theta))/(n-1)
+	
+	S <- SX/m + SY/n
+	L <- c(1,-1)
+	sig <- sqrt(L%*%S%*%L)
+	zscore <- (VR$theta-VS$theta)/sig[1]
+	if (is.nan(zscore) && VR$theta == VR$theta && sig[1] == 0) {
+		zscore <- 0 # special case: no difference between theta's produces a NaN
+	}
+	
+	# Now use those values to calculate the CI as described in 1988 DeLong paper
+	crit_z <- qnorm(1 - ((1 - ci.level)/2))
+	out <- list()
+	out$upper <- qnorm(crit_z) * sig
+	out$lower <- -qnorm(crit_z) * sig
+}
+
 # Calls delongPlacementsCpp safely
 # Ensures that the theta value calculated is correct
 delongPlacements <- function(roc) {

From 9a07ca225111ad839c44e11acd9df0dbced05e27 Mon Sep 17 00:00:00 2001
From: Zane Billings <39701545+wz-billings@users.noreply.github.com>
Date: Tue, 6 Jul 2021 17:01:11 -0400
Subject: [PATCH 03/16] fix: result reporting for CI

---
 R/roc.test.R | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/R/roc.test.R b/R/roc.test.R
index 8e8cdc2..e19f543 100644
--- a/R/roc.test.R
+++ b/R/roc.test.R
@@ -309,8 +309,7 @@ roc.test.roc <- function(roc1, roc2,
 			names(stat) <- "Z"
 			htest$statistic <- stat
 			htest$method <- "DeLong's test for two correlated ROC curves"
-			htest$ci.upper <- stat.ci$upper
-			htest$ci.lower <- stat.ci$lower
+			htest$conf.int <- c(stat.ci$lower, stat.ci$upper)
 			
 			if (alternative == "two.sided")
 				pval <- 2*pnorm(-abs(stat))

From 8d8d2d0a483648ea989eb0a6a9285a4a60c00a89 Mon Sep 17 00:00:00 2001
From: Zane Billings <39701545+wz-billings@users.noreply.github.com>
Date: Tue, 6 Jul 2021 17:11:35 -0400
Subject: [PATCH 04/16] fix: calculates CI correctly

---
 R/delong.R | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/R/delong.R b/R/delong.R
index 16c0e08..a285951 100644
--- a/R/delong.R
+++ b/R/delong.R
@@ -156,8 +156,9 @@ ci.delong.paired <- function(roc1, roc2, ci.level = NULL) {
 	# Now use those values to calculate the CI as described in 1988 DeLong paper
 	crit_z <- qnorm(1 - ((1 - ci.level)/2))
 	out <- list()
-	out$upper <- qnorm(crit_z) * sig
-	out$lower <- -qnorm(crit_z) * sig
+	d <- VR$theta-VS$theta #difference estimate = (1, -1) %*% theta'
+	out$upper <- d + crit_z * sig
+	out$lower <- d - crit_z * sig
 }
 
 # Calls delongPlacementsCpp safely

From adba54cc2ca1de08237bea9e6972c1e561c7894d Mon Sep 17 00:00:00 2001
From: Zane Billings <39701545+wz-billings@users.noreply.github.com>
Date: Tue, 6 Jul 2021 17:15:24 -0400
Subject: [PATCH 05/16] fix: ci.delong.paired now returns correct output

---
 R/delong.R | 1 +
 1 file changed, 1 insertion(+)

diff --git a/R/delong.R b/R/delong.R
index a285951..74f33ea 100644
--- a/R/delong.R
+++ b/R/delong.R
@@ -159,6 +159,7 @@ ci.delong.paired <- function(roc1, roc2, ci.level = NULL) {
 	d <- VR$theta-VS$theta #difference estimate = (1, -1) %*% theta'
 	out$upper <- d + crit_z * sig
 	out$lower <- d - crit_z * sig
+	return(out)
 }
 
 # Calls delongPlacementsCpp safely

From 990495bb8d11ea3f157bb72c7d4c5f7a966b8e8e Mon Sep 17 00:00:00 2001
From: Zane Billings <39701545+wz-billings@users.noreply.github.com>
Date: Tue, 6 Jul 2021 17:29:47 -0400
Subject: [PATCH 06/16] feat: adds conf.level attribute to paired delong test
 output

---
 R/delong.R   | 1 +
 R/roc.test.R | 1 +
 2 files changed, 2 insertions(+)

diff --git a/R/delong.R b/R/delong.R
index 74f33ea..f1f3f41 100644
--- a/R/delong.R
+++ b/R/delong.R
@@ -159,6 +159,7 @@ ci.delong.paired <- function(roc1, roc2, ci.level = NULL) {
 	d <- VR$theta-VS$theta #difference estimate = (1, -1) %*% theta'
 	out$upper <- d + crit_z * sig
 	out$lower <- d - crit_z * sig
+	out$level <- ci.level
 	return(out)
 }
 
diff --git a/R/roc.test.R b/R/roc.test.R
index e19f543..b39f125 100644
--- a/R/roc.test.R
+++ b/R/roc.test.R
@@ -310,6 +310,7 @@ roc.test.roc <- function(roc1, roc2,
 			htest$statistic <- stat
 			htest$method <- "DeLong's test for two correlated ROC curves"
 			htest$conf.int <- c(stat.ci$lower, stat.ci$upper)
+			attr(htest$conf.inf, "conf.level") <- stat.ci$level
 			
 			if (alternative == "two.sided")
 				pval <- 2*pnorm(-abs(stat))

From 2b927ec1f79ab387a78d69140c87a7ca452d4fb6 Mon Sep 17 00:00:00 2001
From: Zane Billings <39701545+wz-billings@users.noreply.github.com>
Date: Tue, 6 Jul 2021 17:42:52 -0400
Subject: [PATCH 07/16] fix: correctly assigns conf.level attribute

---
 R/delong.R   | 4 ++--
 R/roc.test.R | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/R/delong.R b/R/delong.R
index f1f3f41..2911240 100644
--- a/R/delong.R
+++ b/R/delong.R
@@ -157,8 +157,8 @@ ci.delong.paired <- function(roc1, roc2, ci.level = NULL) {
 	crit_z <- qnorm(1 - ((1 - ci.level)/2))
 	out <- list()
 	d <- VR$theta-VS$theta #difference estimate = (1, -1) %*% theta'
-	out$upper <- d + crit_z * sig
-	out$lower <- d - crit_z * sig
+	out$upper <- d + crit_z * sig[1]
+	out$lower <- d - crit_z * sig[1]
 	out$level <- ci.level
 	return(out)
 }
diff --git a/R/roc.test.R b/R/roc.test.R
index b39f125..879ad11 100644
--- a/R/roc.test.R
+++ b/R/roc.test.R
@@ -310,7 +310,7 @@ roc.test.roc <- function(roc1, roc2,
 			htest$statistic <- stat
 			htest$method <- "DeLong's test for two correlated ROC curves"
 			htest$conf.int <- c(stat.ci$lower, stat.ci$upper)
-			attr(htest$conf.inf, "conf.level") <- stat.ci$level
+			attr(htest$conf.int, "conf.level") <- stat.ci$level
 			
 			if (alternative == "two.sided")
 				pval <- 2*pnorm(-abs(stat))

From cc083f71feb988873c53bf8d671f94fb7d215d03 Mon Sep 17 00:00:00 2001
From: Zane Billings <39701545+wz-billings@users.noreply.github.com>
Date: Tue, 6 Jul 2021 17:47:21 -0400
Subject: [PATCH 08/16] fix: adds ability to change confidence level

---
 R/delong.R   | 18 +++++++++---------
 R/roc.test.R |  3 ++-
 2 files changed, 11 insertions(+), 10 deletions(-)

diff --git a/R/delong.R b/R/delong.R
index 2911240..65dc6c7 100644
--- a/R/delong.R
+++ b/R/delong.R
@@ -112,17 +112,17 @@ ci.auc.delong <- function(roc, conf.level) {
 }
 
 # function to calculate the CI
-ci.delong.paired <- function(roc1, roc2, ci.level = NULL) {
+ci.delong.paired <- function(roc1, roc2, conf.level = NULL) {
 	
 	# check whether CI level was specified correctly or not.
-	if (is.null(ci.level)) {
-		ci.level <- 0.95
+	if (is.null(conf.level)) {
+		conf.level <- 0.95
 		message("ci.level not specified, defaults to 0.95.")
-	} else if (0 > ci.level | 1 < ci.level) {
-		ci.level <- 0.95
+	} else if (0 > conf.level | 1 < conf.level) {
+		conf.level <- 0.95
 		message("ci.level must be between 0 and 1. defaulting to 0.95.")
-	} else if (!is.numeric(ci.level)) {
-		ci.level <- 0.95
+	} else if (!is.numeric(conf.level)) {
+		conf.level <- 0.95
 		message("ci.level must be numeric between 0 and 1. defaulting to 0.95.")
 	}
 	
@@ -154,12 +154,12 @@ ci.delong.paired <- function(roc1, roc2, ci.level = NULL) {
 	}
 	
 	# Now use those values to calculate the CI as described in 1988 DeLong paper
-	crit_z <- qnorm(1 - ((1 - ci.level)/2))
+	crit_z <- qnorm(1 - ((1 - conf.level)/2))
 	out <- list()
 	d <- VR$theta-VS$theta #difference estimate = (1, -1) %*% theta'
 	out$upper <- d + crit_z * sig[1]
 	out$lower <- d - crit_z * sig[1]
-	out$level <- ci.level
+	out$level <- conf.level
 	return(out)
 }
 
diff --git a/R/roc.test.R b/R/roc.test.R
index 879ad11..5d74126 100644
--- a/R/roc.test.R
+++ b/R/roc.test.R
@@ -116,6 +116,7 @@ roc.test.roc <- function(roc1, roc2,
 						 ties.method="first",
 						 progress=getOption("pROCProgress")$name,
 						 parallel=FALSE,
+						 conf.level,
 						 ...) {
 	alternative <- match.arg(alternative)
 	data.names <- paste(deparse(substitute(roc1)), "and", deparse(substitute(roc2)))
@@ -305,7 +306,7 @@ roc.test.roc <- function(roc1, roc2,
 	if (method == "delong") {
 		if (paired) {
 			stat <- delong.paired.test(roc1, roc2)
-			stat.ci <- ci.delong.paired(roc1, roc2)
+			stat.ci <- ci.delong.paired(roc1, roc2, conf.level)
 			names(stat) <- "Z"
 			htest$statistic <- stat
 			htest$method <- "DeLong's test for two correlated ROC curves"

From 05139f6539be1e40f80d5f84ca854951a2496473 Mon Sep 17 00:00:00 2001
From: Zane Billings <39701545+wz-billings@users.noreply.github.com>
Date: Tue, 6 Jul 2021 17:50:05 -0400
Subject: [PATCH 09/16] feat: conf.level argument now has a default.

---
 R/roc.test.R | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/R/roc.test.R b/R/roc.test.R
index 5d74126..4142709 100644
--- a/R/roc.test.R
+++ b/R/roc.test.R
@@ -116,7 +116,7 @@ roc.test.roc <- function(roc1, roc2,
 						 ties.method="first",
 						 progress=getOption("pROCProgress")$name,
 						 parallel=FALSE,
-						 conf.level,
+						 conf.level=NULL,
 						 ...) {
 	alternative <- match.arg(alternative)
 	data.names <- paste(deparse(substitute(roc1)), "and", deparse(substitute(roc2)))

From f7bd02c303455af1122f02e945d35ece41c0edc5 Mon Sep 17 00:00:00 2001
From: Zane Billings <39701545+wz-billings@users.noreply.github.com>
Date: Wed, 7 Jul 2021 12:12:00 -0400
Subject: [PATCH 10/16] docs: added arg conf.level to roc.test docs.

---
 man/roc.test.Rd | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/man/roc.test.Rd b/man/roc.test.Rd
index fbcaee4..2cefbd4 100644
--- a/man/roc.test.Rd
+++ b/man/roc.test.Rd
@@ -25,7 +25,7 @@
 specificity = NULL, alternative = c("two.sided", "less", "greater"),
 paired=NULL, reuse.auc=TRUE, boot.n=2000, boot.stratified=TRUE,
 ties.method="first", progress=getOption("pROCProgress")$name,
-parallel=FALSE, ...)
+parallel=FALSE, conf.level=0.95, ...)
 \S3method{roc.test}{auc}(roc1, roc2, ...)
 \S3method{roc.test}{smooth.roc}(roc1, roc2, ...)
 \S3method{roc.test}{formula}(formula, data, ...)
@@ -97,6 +97,8 @@ na.rm=TRUE, method=NULL, ...)
   \item{parallel}{if TRUE, the bootstrap is processed in parallel, using 
     parallel backend provided by plyr (foreach).
   }
+  \item{conf.level}{a numeric scalar between 0 and 1 (non-inclusive) which
+    species the confidence level to use for any calculated CI's.}
   \item{\dots}{further arguments passed to or from other methods,
     especially arguments for \code{\link{roc}} and \code{roc.test.roc}
     when calling \code{roc.test.default} or \code{roc.test.formula}.

From 5b2b455870a1141d86a8b9b05a0c153da13d4f5a Mon Sep 17 00:00:00 2001
From: Zane Billings <39701545+wz-billings@users.noreply.github.com>
Date: Wed, 7 Jul 2021 12:18:41 -0400
Subject: [PATCH 11/16] refactor: moved conf.level checks to roc.test.R.

---
 R/roc.test.R | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/R/roc.test.R b/R/roc.test.R
index 4142709..82a3434 100644
--- a/R/roc.test.R
+++ b/R/roc.test.R
@@ -116,7 +116,7 @@ roc.test.roc <- function(roc1, roc2,
 						 ties.method="first",
 						 progress=getOption("pROCProgress")$name,
 						 parallel=FALSE,
-						 conf.level=NULL,
+						 conf.level=0.95,
 						 ...) {
 	alternative <- match.arg(alternative)
 	data.names <- paste(deparse(substitute(roc1)), "and", deparse(substitute(roc2)))
@@ -250,6 +250,15 @@ roc.test.roc <- function(roc1, roc2,
 			}
 			if (roc1$direction != roc2$direction)
 				warning("DeLong's test should not be applied to ROC curves with a different direction.")
+			
+			# Check if conf.level is specified correctly. This is currently
+			# only used for the delong paired method, which is why it lives 
+			# here for now.
+			if (!is.numeric(conf.level)) {
+				stop("conf.level must be numeric between 0 and 1.")
+			} else if (0 > conf.level | 1 < conf.level) {
+				stop("conf.level must be between 0 and 1.")
+			}
 		}
 		else if (method == "venkatraman") {
 			if (has.partial.auc(roc1))

From 9b0a687154ab2214897500ab15aff2a28a5ad551 Mon Sep 17 00:00:00 2001
From: Zane Billings <39701545+wz-billings@users.noreply.github.com>
Date: Wed, 7 Jul 2021 12:32:41 -0400
Subject: [PATCH 12/16] refactor: paired delong calculations are no longer
 repeated.

---
 R/delong.R | 77 ++++++++++++++++++------------------------------------
 1 file changed, 26 insertions(+), 51 deletions(-)

diff --git a/R/delong.R b/R/delong.R
index 65dc6c7..13573b9 100644
--- a/R/delong.R
+++ b/R/delong.R
@@ -18,31 +18,15 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 # Delong's test paired, used by roc.test.roc
-delong.paired.test <- function(roc1, roc2) {
-  n <- length(roc1$controls)
-  m <- length(roc1$cases)
-
-  VR <- delongPlacements(roc1)
-  VS <- delongPlacements(roc2)
-
-  SX <- matrix(NA, ncol=2, nrow=2)
-  SX[1,1] <- sum((VR$X - VR$theta) * (VR$X - VR$theta))/(m-1)
-  SX[1,2] <- sum((VR$X - VR$theta) * (VS$X - VS$theta))/(m-1)
-  SX[2,1] <- sum((VS$X - VS$theta) * (VR$X - VR$theta))/(m-1)
-  SX[2,2] <- sum((VS$X - VS$theta) * (VS$X - VS$theta))/(m-1)
-
-  SY <- matrix(NA, ncol=2, nrow=2)
-  SY[1,1] <- sum((VR$Y - VR$theta) * (VR$Y - VR$theta))/(n-1)
-  SY[1,2] <- sum((VR$Y - VR$theta) * (VS$Y - VS$theta))/(n-1)
-  SY[2,1] <- sum((VS$Y - VS$theta) * (VR$Y - VR$theta))/(n-1)
-  SY[2,2] <- sum((VS$Y - VS$theta) * (VS$Y - VS$theta))/(n-1)
-
-  S <- SX/m + SY/n
-  L <- c(1,-1)
-  sig <- sqrt(L%*%S%*%L)
-  zscore <- (VR$theta-VS$theta)/sig[1]
-  if (is.nan(zscore) && VR$theta == VR$theta && sig[1] == 0)
-    zscore <- 0 # special case: no difference between theta's produces a NaN
+delong.paired.test <- function(calcs) {
+	
+	# Input calcs is a list returned by delong.paired.calculations().
+	
+	zscore <- with(calcs, d/sig)
+	
+	if (is.nan(zscore) && calc$d == 0 && calcs$sig == 0)
+		zscore <- 0 # special case: no difference between theta's produces a NaN
+	
   return(zscore)
 }
 
@@ -112,21 +96,23 @@ ci.auc.delong <- function(roc, conf.level) {
 }
 
 # function to calculate the CI
-ci.delong.paired <- function(roc1, roc2, conf.level = NULL) {
+ci.delong.paired <- function(calcs, conf.level) {
 	
-	# check whether CI level was specified correctly or not.
-	if (is.null(conf.level)) {
-		conf.level <- 0.95
-		message("ci.level not specified, defaults to 0.95.")
-	} else if (0 > conf.level | 1 < conf.level) {
-		conf.level <- 0.95
-		message("ci.level must be between 0 and 1. defaulting to 0.95.")
-	} else if (!is.numeric(conf.level)) {
-		conf.level <- 0.95
-		message("ci.level must be numeric between 0 and 1. defaulting to 0.95.")
-	}
+	# Input calcs is a list generated by delong.paired.calculations().
+	# CI is calculated using the normally distributed pivot given in
+	# DeLong's 1988 paper.
 	
-	# Code from delong test for intermediate calculations
+	crit_z <- qnorm(1 - ((1 - conf.level)/2))
+	out <- list()
+	out$upper <- with(calcs, d + crit_z * sig)
+	out$lower <- with(calcs, d - crit_z * sig)
+	out$level <- conf.level
+	return(out)
+}
+
+# Runs the placements and main calculations for the paired DeLong's test
+#  so that they can be easily used by both the test and CI functions.
+delong.paired.calculations <- function(roc1, roc2) {
 	n <- length(roc1$controls)
 	m <- length(roc1$cases)
 	
@@ -148,19 +134,8 @@ ci.delong.paired <- function(roc1, roc2, conf.level = NULL) {
 	S <- SX/m + SY/n
 	L <- c(1,-1)
 	sig <- sqrt(L%*%S%*%L)
-	zscore <- (VR$theta-VS$theta)/sig[1]
-	if (is.nan(zscore) && VR$theta == VR$theta && sig[1] == 0) {
-		zscore <- 0 # special case: no difference between theta's produces a NaN
-	}
-	
-	# Now use those values to calculate the CI as described in 1988 DeLong paper
-	crit_z <- qnorm(1 - ((1 - conf.level)/2))
-	out <- list()
-	d <- VR$theta-VS$theta #difference estimate = (1, -1) %*% theta'
-	out$upper <- d + crit_z * sig[1]
-	out$lower <- d - crit_z * sig[1]
-	out$level <- conf.level
-	return(out)
+	d <- VR$theta - VS$theta
+	return(list("d" = d, "sig" = sig[[1]]))
 }
 
 # Calls delongPlacementsCpp safely

From b13f8192c058b574fc08b8144fe58c3e72e0d4d0 Mon Sep 17 00:00:00 2001
From: Zane Billings <39701545+wz-billings@users.noreply.github.com>
Date: Wed, 7 Jul 2021 12:33:46 -0400
Subject: [PATCH 13/16] refactor: roc.test now calls delong.paired.calculations

---
 R/roc.test.R | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/R/roc.test.R b/R/roc.test.R
index 82a3434..8511cf3 100644
--- a/R/roc.test.R
+++ b/R/roc.test.R
@@ -314,8 +314,9 @@ roc.test.roc <- function(roc1, roc2,
 	
 	if (method == "delong") {
 		if (paired) {
-			stat <- delong.paired.test(roc1, roc2)
-			stat.ci <- ci.delong.paired(roc1, roc2, conf.level)
+			delong.calcs <- delong.paired.calculations(roc1, roce)
+			stat <- delong.paired.test(delong.calcs)
+			stat.ci <- ci.delong.paired(delong.calcs, conf.level)
 			names(stat) <- "Z"
 			htest$statistic <- stat
 			htest$method <- "DeLong's test for two correlated ROC curves"

From 025c1a5eeef59ca6ac47cb188eb536d08899abde Mon Sep 17 00:00:00 2001
From: Zane Billings <39701545+wz-billings@users.noreply.github.com>
Date: Wed, 7 Jul 2021 12:36:45 -0400
Subject: [PATCH 14/16] fix: roce -> roc2. oops

---
 R/roc.test.R | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/R/roc.test.R b/R/roc.test.R
index 8511cf3..9df2d37 100644
--- a/R/roc.test.R
+++ b/R/roc.test.R
@@ -314,7 +314,7 @@ roc.test.roc <- function(roc1, roc2,
 	
 	if (method == "delong") {
 		if (paired) {
-			delong.calcs <- delong.paired.calculations(roc1, roce)
+			delong.calcs <- delong.paired.calculations(roc1, roc2)
 			stat <- delong.paired.test(delong.calcs)
 			stat.ci <- ci.delong.paired(delong.calcs, conf.level)
 			names(stat) <- "Z"

From ebf313eaed1eba8bc47fe635bb2b1ba982e2cd5c Mon Sep 17 00:00:00 2001
From: Zane Billings <39701545+wz-billings@users.noreply.github.com>
Date: Mon, 19 Jul 2021 14:00:54 -0400
Subject: [PATCH 15/16] docs: added conf.int to value in roc.test docs

---
 man/roc.test.Rd | 1 +
 1 file changed, 1 insertion(+)

diff --git a/man/roc.test.Rd b/man/roc.test.Rd
index 2cefbd4..80443e4 100644
--- a/man/roc.test.Rd
+++ b/man/roc.test.Rd
@@ -302,6 +302,7 @@ na.rm=TRUE, method=NULL, ...)
   \item{statistic}{the value of the Z (\code{method="delong"}) or D
     (\code{method="bootstrap"}) statistics.
   }
+  \item{conf.int}{the confidence interval of the test (currently only returned for the paired DeLong's test). Has an attribute \code{conf.level} specifiying the level of the test.}
   \item{alternative}{the alternative hypothesis.}
   \item{method}{the character string \dQuote{DeLong's test for two
       correlated ROC curves} (if \code{method="delong"}) or

From 4cb8634497bf587c45d2d65d589182896faf149c Mon Sep 17 00:00:00 2001
From: Zane Billings <39701545+wz-billings@users.noreply.github.com>
Date: Mon, 19 Jul 2021 15:50:28 -0400
Subject: [PATCH 16/16] tests: added tests for paired delong confidence
 interval.

---
 tests/testthat/test-roc.test.R | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/tests/testthat/test-roc.test.R b/tests/testthat/test-roc.test.R
index cf89377..83d5b82 100644
--- a/tests/testthat/test-roc.test.R
+++ b/tests/testthat/test-roc.test.R
@@ -16,21 +16,30 @@ test_that("roc.test works", {
 test_that("roc.test statistic and p are as expected with defaults", {
 	expect_equal(t1$statistic, c(Z=2.20898359144091))
 	expect_equal(t1$p.value, 0.0271757822291882)
+	expect_equal(t1$conf.int[[1]], 0.0104061769564846)
+	expect_equal(t1$conf.int[[2]], 0.174214419249478)
 	expect_match(t1$method, "DeLong")
 	expect_match(t1$method, "correlated")
 	expect_identical(t1$alternative, "two.sided")
+	expect_identical(attr(t1$conf.int, "conf.level"), 0.95)
 	
 	expect_equal(t2$statistic, c(Z=2.79777591868904))
 	expect_equal(t2$p.value, 0.00514557970691098)
+	expect_equal(t2$conf.int[[1]], 0.0634011709339876)
+	expect_equal(t2$conf.int[[2]], 0.3600405634833566)
 	expect_match(t2$method, "DeLong")
 	expect_match(t2$method, "correlated")
 	expect_identical(t2$alternative, "two.sided")
+	expect_identical(attr(t2$conf.int, "conf.level"), 0.95)
 	
 	expect_equal(t3$statistic, c(Z=-1.39077002573558))
 	expect_equal(t3$p.value, 0.164295175223054)
+	expect_equal(t3$conf.int[[1]], -0.2876917446341914)
+	expect_equal(t3$conf.int[[2]], 0.0488706064228094)
 	expect_match(t3$method, "DeLong")
 	expect_match(t3$method, "correlated")
 	expect_identical(t3$alternative, "two.sided")
+	expect_identical(attr(t3$conf.int, "conf.level"), 0.95)
 })
 
 test_that("two.sided roc.test produces identical p values when roc curves are reversed", {