Add BigQuery dependency, BigQueryUtils, move to unshaded google-cloud-java, and newer Spark/Hadoop/Guava

build.gradle

@@ -14,7 +14,7 @@ plugins {
     id 'signing'
     id "jacoco"
     id "de.undercouch.download" version "2.1.0" //used for downloading GSA lib
-    id "com.github.johnrengelman.shadow" version "1.2.3"    //used to build the shadow and sparkJars
+    id "com.github.johnrengelman.shadow" version "5.0.0"    //used to build the shadow and sparkJars
     id "com.github.ben-manes.versions" version "0.12.0" //used for identifying dependencies that need updating
     id 'com.palantir.git-version' version '0.5.1' //version helper
 }
@@ -61,15 +61,17 @@ final requiredJavaVersion = "8"
 final htsjdkVersion = System.getProperty('htsjdk.version','2.19.0')
 final picardVersion = System.getProperty('picard.version','2.19.0')
 final barclayVersion = System.getProperty('barclay.version','2.1.0')
-final sparkVersion = System.getProperty('spark.version', '2.2.0')
-final hadoopVersion = System.getProperty('hadoop.version', '2.8.2')
+final sparkVersion = System.getProperty('spark.version', '2.4.3')
+final hadoopVersion = System.getProperty('hadoop.version', '3.2.0')
 final disqVersion = System.getProperty('disq.version','0.3.0')
 final genomicsdbVersion = System.getProperty('genomicsdb.version','1.0.3')
+final bigQueryVersion = System.getProperty('bigQuery.version', '1.69.0')
+final guavaVersion = System.getProperty('guava.version', '27.1-jre')
 final testNGVersion = '6.11'
 // Using the shaded version to avoid conflicts between its protobuf dependency
 // and that of Hadoop/Spark (either the one we reference explicitly, or the one
 // provided by dataproc).
-final googleCloudNioDependency = 'com.google.cloud:google-cloud-nio:0.81.0-alpha:shaded'
+final googleCloudNioDependency = 'com.google.cloud:google-cloud-nio:0.87.0-alpha'
 
 final baseJarName = 'gatk'
 final secondaryBaseJarName = 'hellbender'
@@ -148,7 +150,7 @@ ensureBuildPrerequisites(requiredJavaVersion, largeResourcesFolder, buildPrerequ
 configurations.all {
     resolutionStrategy {
         // the snapshot folder contains a dev version of guava, we don't want to use that.
-        force 'com.google.guava:guava:18.0'
+        force 'com.google.guava:guava:' + guavaVersion
         // force the htsjdk version so we don't get a different one transitively
         force 'com.github.samtools:htsjdk:' + htsjdkVersion
         // later versions explode Hadoop
@@ -157,6 +159,10 @@ configurations.all {
         force 'org.testng:testng:' + testNGVersion
         force 'org.broadinstitute:barclay:' + barclayVersion
 
+        force 'com.fasterxml.jackson.core:jackson-core:2.6.7'
+        force 'com.fasterxml.jackson.core:jackson-databind:2.6.7'
+        force 'com.fasterxml.jackson.core:jackson-annotations:2.6.7'
+
         // make sure we don't pick up an incorrect version of the GATK variant of the google-nio library
         // via Picard, etc.
         force googleCloudNioDependency
@@ -233,9 +239,11 @@ dependencies {
         exclude module: 'protobuf-java'
     }
     compile 'com.opencsv:opencsv:3.4'
-    compile 'com.google.guava:guava:18.0'
+    compile 'com.google.guava:guava:' + guavaVersion
     compile 'com.github.samtools:htsjdk:'+ htsjdkVersion
     compile googleCloudNioDependency
+
+    compile 'com.google.cloud:google-cloud-bigquery:' + bigQueryVersion
 
     compile "gov.nist.math.jama:gov.nist.math.jama:1.1.1"
 
@@ -385,15 +393,15 @@ tasks.withType(Jar) {
     }
 }
 
-task wrapper(type: Wrapper) {
-    gradleVersion = '3.1'
+wrapper {
+    gradleVersion = '5.4.1'
 }
 
 tasks.withType(ShadowJar) {
     from(project.sourceSets.main.output)
     baseName = project.name + '-package'
     mergeServiceFiles()
-    relocate 'com.google.common', 'org.broadinstitute.hellbender.relocated.com.google.common'
+    // relocate 'com.google.common', 'org.broadinstitute.hellbender.relocated.com.google.common'
     zip64 true
     exclude 'log4j.properties' // from adam jar as it clashes with hellbender's log4j2.xml
     exclude '**/*.SF' // these are Manifest signature files and
@@ -633,6 +641,9 @@ task gatkDoc(type: Javadoc, dependsOn: classes) {
     // other resources, but we only want the files that javadoc can handle, so just take the .java files.
     source = sourceSets.main.allJava + files(configurations.externalSourceConfiguration.collect { zipTree(it) })
     include '**/*.java'
+
+    // TODO: temporary hack until we move Picard to a non-shaded google-cloud-nio version
+    exclude '**/GoogleStorageUtils.java'
 
     // The gatkDoc process instantiates any documented feature classes, so to run it we need the entire
     // runtime classpath, as well as jdk javadoc files such as tools.jar, where com.sun.javadoc lives.
@@ -670,6 +681,9 @@ task gatkTabComplete(type: Javadoc, dependsOn: classes) {
     source = sourceSets.main.allJava + files(configurations.externalSourceConfiguration.collect { zipTree(it) })
     include '**/*.java'
 
+    // TODO: temporary hack until we move Picard to a non-shaded google-cloud-nio version
+    exclude '**/GoogleStorageUtils.java'
+
     // The gatkDoc process instantiates any documented feature classes, so to run it we need the entire
     // runtime classpath, as well as jdk javadoc files such as tools.jar, where com.sun.javadoc lives, and Picard.
     classpath = sourceSets.main.runtimeClasspath + javadocJDKFiles

gradle/wrapper/gradle-wrapper.properties

@@ -3,4 +3,4 @@ distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-3.1-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-5.4.1-bin.zip

scripts/docker/dockertest.gradle

@@ -62,7 +62,7 @@ task testOnPackagedReleaseJar(type: Test){
     }
 
     classpath = files( gatkJar, testDependencyJar, testClassesJar)
-    testClassesDir = file(testClassesUnpacked)
+    testClassesDirs = files(testClassesUnpacked)
 }
 
 // Task intended to collect coverage data from testOnPackagedReleaseJar executed inside the docker image