From 94a71cc1f0894b51c53371c1bab5d2955f7e01ef Mon Sep 17 00:00:00 2001 From: Khalid Shakir Date: Sun, 9 May 2021 19:46:33 -0400 Subject: [PATCH 1/2] Database patches: - Wait longer for containers to download and startup - Use the latest PostgreSQL jdbc driver - Updated liquibase and schema comparison tests - Empty LOBs are always stored as null - Fixed test description for empty lobs - Test the various databases with centaur local - Don't start a db when CI checking publishing - Help liquibase by using valid location of S3FSP - Longer leeway for logging events for longer liquibase --- .travis.yml | 10 +-- .../database/slick/SlickDatabase.scala | 2 +- .../cromwell/database/sql/SqlConverters.scala | 12 ++-- .../java.nio.file.spi.FileSystemProvider | 2 +- project/Dependencies.scala | 18 ++--- .../scala/cromwell/CromwellTestKitSpec.scala | 2 +- .../services/database/DatabaseTestKit.scala | 16 +++-- .../database/LiquibaseComparisonSpec.scala | 71 ++++++++++++------- .../cromwell/services/database/LobSpec.scala | 12 ++-- .../keyvalue/impl/KeyValueDatabaseSpec.scala | 3 +- src/ci/bin/test.inc.sh | 4 ++ 11 files changed, 86 insertions(+), 66 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3c098c36611..de4c5e9920d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -40,21 +40,18 @@ env: - >- BUILD_TYPE=centaurHoricromtalPapiV2beta BUILD_MYSQL=5.7 - - >- - BUILD_TYPE=centaurHoricromtalPapiV2beta - BUILD_MARIADB=10.3 - >- BUILD_TYPE=centaurHoricromtalEngineUpgradePapiV2alpha1 BUILD_MYSQL=5.7 - - >- - BUILD_TYPE=centaurHoricromtalEngineUpgradePapiV2alpha1 - BUILD_MARIADB=10.3 - >- BUILD_TYPE=centaurPapiUpgradePapiV2alpha1 BUILD_MYSQL=5.7 - >- BUILD_TYPE=centaurPapiUpgradeNewWorkflowsPapiV2alpha1 BUILD_MYSQL=5.7 + - >- + BUILD_TYPE=centaurLocal + BUILD_MARIADB=10.3 - >- BUILD_TYPE=centaurLocal BUILD_MYSQL=5.7 @@ -78,7 +75,6 @@ env: BUILD_MYSQL=5.7 - >- BUILD_TYPE=checkPublish - BUILD_MYSQL=5.7 - >- BUILD_TYPE=conformanceLocal BUILD_MYSQL=5.7 diff --git a/database/sql/src/main/scala/cromwell/database/slick/SlickDatabase.scala b/database/sql/src/main/scala/cromwell/database/slick/SlickDatabase.scala index 037fd0fe540..e54974f6618 100644 --- a/database/sql/src/main/scala/cromwell/database/slick/SlickDatabase.scala +++ b/database/sql/src/main/scala/cromwell/database/slick/SlickDatabase.scala @@ -228,7 +228,7 @@ abstract class SlickDatabase(override val originalDatabaseConfig: Config) extend Instead resorting to reflection. */ val message = pSQLException.getServerErrorMessage - val field = classOf[ServerErrorMessage].getDeclaredField("m_mesgParts") + val field = classOf[ServerErrorMessage].getDeclaredField("mesgParts") field.setAccessible(true) val parts = field.get(message).asInstanceOf[java.util.Map[Character, String]] parts.remove('D') diff --git a/database/sql/src/main/scala/cromwell/database/sql/SqlConverters.scala b/database/sql/src/main/scala/cromwell/database/sql/SqlConverters.scala index fe1bf5f7dba..6d734bdea94 100644 --- a/database/sql/src/main/scala/cromwell/database/sql/SqlConverters.scala +++ b/database/sql/src/main/scala/cromwell/database/sql/SqlConverters.scala @@ -14,11 +14,12 @@ object SqlConverters { // https://github.com/slick/slick/issues/1026 implicit class TimestampToSystemOffsetDateTime(val timestamp: Timestamp) extends AnyVal { - def toSystemOffsetDateTime = timestamp.toLocalDateTime.atZone(ZoneId.systemDefault).toOffsetDateTime + def toSystemOffsetDateTime: OffsetDateTime = timestamp.toLocalDateTime.atZone(ZoneId.systemDefault).toOffsetDateTime } implicit class OffsetDateTimeToSystemTimestamp(val offsetDateTime: OffsetDateTime) extends AnyVal { - def toSystemTimestamp = Timestamp.valueOf(offsetDateTime.atZoneSameInstant(ZoneId.systemDefault).toLocalDateTime) + def toSystemTimestamp: Timestamp = + Timestamp.valueOf(offsetDateTime.atZoneSameInstant(ZoneId.systemDefault).toLocalDateTime) } implicit class ClobOptionToRawString(val clobOption: Option[Clob]) extends AnyVal { @@ -56,10 +57,11 @@ object SqlConverters { import eu.timepit.refined.api.Refined import eu.timepit.refined.collection.NonEmpty - def toClobOption: Option[SerialClob] = if (str.isEmpty) None else Option(new SerialClob(str.toCharArray)) + def toClobOption: Option[SerialClob] = + if (str == null || str.isEmpty) None else Option(new SerialClob(str.toCharArray)) def toClob(default: String Refined NonEmpty): SerialClob = { - val nonEmpty = if (str.isEmpty) default.value else str + val nonEmpty = if (str == null || str.isEmpty) default.value else str new SerialClob(nonEmpty.toCharArray) } } @@ -91,7 +93,7 @@ object SqlConverters { } implicit class BytesToBlobOption(val bytes: Array[Byte]) extends AnyVal { - def toBlobOption: Option[SerialBlob] = if (bytes.isEmpty) None else Option(new SerialBlob(bytes)) + def toBlobOption: Option[SerialBlob] = if (bytes == null || bytes.isEmpty) None else Option(new SerialBlob(bytes)) } implicit class EnhancedFiniteDuration(val duration: FiniteDuration) extends AnyVal { diff --git a/filesystems/s3/src/main/resources/META-INF/services/java.nio.file.spi.FileSystemProvider b/filesystems/s3/src/main/resources/META-INF/services/java.nio.file.spi.FileSystemProvider index 3a259796ab2..66974158b24 100644 --- a/filesystems/s3/src/main/resources/META-INF/services/java.nio.file.spi.FileSystemProvider +++ b/filesystems/s3/src/main/resources/META-INF/services/java.nio.file.spi.FileSystemProvider @@ -1,2 +1,2 @@ # if not present, FileSystems.newFileSystem throw NotProviderFoundException -com.upplication.s3fs.S3FileSystemProvider +org.lerch.s3fs.S3FileSystemProvider diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 6cad93065ce..7a39b111e20 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -58,7 +58,6 @@ object Dependencies { private val hsqldbV = "2.5.1" private val http4sVersion = "0.21.7" // scala-steward:off (CROM-6678) private val jacksonV = "2.12.2" - private val jacksonJqV = "1.0.0-preview.20201123" private val janinoV = "3.1.2" private val javaxActivationV = "1.2.0" // jaxb-impl 2.3.3 depends on com.sun.activation:jakarta.activation and jakarta.xml.bind:jakarta.xml.bind-api, @@ -73,12 +72,7 @@ object Dependencies { private val kindProjectorV = "0.9.9" private val kittensV = "2.2.1" private val liquibaseSlf4jV = "4.0.0" - // Scala Steward wanted to upgrade liquibase-core to 3.10.2 but that version does not find some uniqueness - // constraints and models datatypes in ways that are incompatible with our test expectations. - // liquibase-core 4.0.0 did not have either of those problems but produced tons of strange warnings at runtime - // similar in form to this: https://github.com/liquibase/liquibase/issues/1294 - // Pinning Liquibase version for the time being. - private val liquibaseV = "3.6.3" // scala-steward:off + private val liquibaseV = "4.3.5" private val logbackV = "1.2.3" private val lz4JavaV = "1.7.1" private val mariadbV = "2.7.0" @@ -92,9 +86,7 @@ object Dependencies { private val owlApiV = "5.1.16" // scala-steward: off (CROM-6677) private val paradiseV = "2.1.1" private val pegdownV = "1.6.0" - // For org.postgresql:postgresql 42.2.6 - 42.2.14: - // java.lang.NoSuchFieldException: m_mesgParts in KeyValueSpec "fail if one of the inserts fails" - private val postgresV = "42.2.5" // scala-steward:off + private val postgresV = "42.2.20" private val rdf4jV = "2.4.2" private val refinedV = "0.9.18" private val rhinoV = "1.7.13" @@ -244,10 +236,8 @@ object Dependencies { ) private val liquibaseDependencies = List( - "org.liquibase" % "liquibase-core" % liquibaseV, - // The exclusion below will be needed if / when liquibase-core is upgraded to 3.10+ - // Avert collision with jakarta.xml.bind-api - // exclude("javax.xml.bind", "jaxb-api"), + "org.liquibase" % "liquibase-core" % liquibaseV + exclude("javax.xml.bind", "jaxb-api"), // This is to stop liquibase from being so noisy by default // See: http://stackoverflow.com/questions/20880783/how-to-get-liquibase-to-log-using-slf4j "com.mattbertolini" % "liquibase-slf4j" % liquibaseSlf4jV diff --git a/server/src/test/scala/cromwell/CromwellTestKitSpec.scala b/server/src/test/scala/cromwell/CromwellTestKitSpec.scala index 32216b77239..ed94778c6be 100644 --- a/server/src/test/scala/cromwell/CromwellTestKitSpec.scala +++ b/server/src/test/scala/cromwell/CromwellTestKitSpec.scala @@ -94,7 +94,7 @@ object CromwellTestKitSpec { | # Some of our tests fire off a message, then expect a particular event message within 3s (the default). | # Especially on CI, the metadata test does not seem to be returning in time. So, overriding the timeouts | # with slightly higher values. Alternatively, could also adjust the akka.test.timefactor only in CI. - | filter-leeway = 10s + | filter-leeway = 20s | single-expect-default = 5s | default-timeout = 10s | } diff --git a/services/src/test/scala/cromwell/services/database/DatabaseTestKit.scala b/services/src/test/scala/cromwell/services/database/DatabaseTestKit.scala index f8ce024fd2f..476636e3de3 100644 --- a/services/src/test/scala/cromwell/services/database/DatabaseTestKit.scala +++ b/services/src/test/scala/cromwell/services/database/DatabaseTestKit.scala @@ -1,9 +1,7 @@ package cromwell.services.database -import java.sql.Connection - import better.files._ -import com.dimafeng.testcontainers.{Container, JdbcDatabaseContainer, MariaDBContainer, MySQLContainer, PostgreSQLContainer} +import com.dimafeng.testcontainers._ import com.typesafe.config.{Config, ConfigFactory} import com.typesafe.scalalogging.StrictLogging import cromwell.database.migration.liquibase.LiquibaseUtils @@ -12,12 +10,14 @@ import cromwell.services.ServicesStore.EnhancedSqlDatabase import cromwell.services.{EngineServicesStore, MetadataServicesStore} import liquibase.snapshot.DatabaseSnapshot import liquibase.structure.core.Index +import org.testcontainers.containers.{JdbcDatabaseContainer => JavaJdbcDatabaseContainer} import org.testcontainers.utility.DockerImageName import slick.jdbc.JdbcProfile import slick.jdbc.meta.{MIndexInfo, MPrimaryKey} +import java.sql.Connection import scala.concurrent.Await -import scala.concurrent.duration.Duration +import scala.concurrent.duration._ object DatabaseTestKit extends StrictLogging { @@ -109,7 +109,7 @@ object DatabaseTestKit extends StrictLogging { } def getDatabaseTestContainer(databaseSystem: DatabaseSystem): Option[Container] = { - databaseSystem match { + val containerOption: Option[SingleContainer[_ <: JavaJdbcDatabaseContainer[_]]] = databaseSystem match { case HsqldbDatabaseSystem => None case networkDbSystem: NetworkDatabaseSystem => networkDbSystem.platform match { @@ -135,6 +135,12 @@ object DatabaseTestKit extends StrictLogging { case _ => None } } + + // Give slow CI like TravisCI a bit more patience to pull and then start the container + containerOption.map(_.configure { container => + container.withStartupTimeoutSeconds(5.minutes.toSeconds.toInt) + () + }) } def initializeDatabaseByContainerOptTypeAndSystem[A <: SlickDatabase](containerOpt: Option[Container], diff --git a/services/src/test/scala/cromwell/services/database/LiquibaseComparisonSpec.scala b/services/src/test/scala/cromwell/services/database/LiquibaseComparisonSpec.scala index 45d7aca64e5..2b292c871ac 100644 --- a/services/src/test/scala/cromwell/services/database/LiquibaseComparisonSpec.scala +++ b/services/src/test/scala/cromwell/services/database/LiquibaseComparisonSpec.scala @@ -25,16 +25,18 @@ import scala.reflect._ */ class LiquibaseComparisonSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers with ScalaFutures { - implicit val executionContext = ExecutionContext.global + implicit val executionContext: ExecutionContext = ExecutionContext.global - implicit val defaultPatience = PatienceConfig(timeout = scaled(5.seconds), interval = scaled(100.millis)) + implicit val defaultPatience: PatienceConfig = + PatienceConfig(timeout = scaled(5.seconds), interval = scaled(100.millis)) CromwellDatabaseType.All foreach { databaseType => + lazy val expectedSnapshot = DatabaseTestKit.inMemorySnapshot(databaseType, SlickSchemaManager) - lazy val expectedColumns = get[Column](expectedSnapshot) - lazy val expectedPrimaryKeys = get[PrimaryKey](expectedSnapshot) - lazy val expectedForeignKeys = get[ForeignKey](expectedSnapshot) - lazy val expectedUniqueConstraints = get[UniqueConstraint](expectedSnapshot) + lazy val expectedColumns = get[Column](expectedSnapshot).sorted + lazy val expectedPrimaryKeys = get[PrimaryKey](expectedSnapshot).sorted + lazy val expectedForeignKeys = get[ForeignKey](expectedSnapshot).sorted + lazy val expectedUniqueConstraints = get[UniqueConstraint](expectedSnapshot).sorted lazy val expectedIndexes = get[Index](expectedSnapshot) filterNot DatabaseTestKit.isGenerated DatabaseSystem.All foreach { databaseSystem => @@ -80,7 +82,8 @@ class LiquibaseComparisonSpec extends AnyFlatSpec with CromwellTimeoutSpec with // Auto increment columns may have different types, such as SERIAL/BIGSERIAL // https://www.postgresql.org/docs/11/datatype-numeric.html#DATATYPE-SERIAL val actualColumnDefault = ColumnDefault(actualColumnType, actualColumn.getDefaultValue) - val autoIncrementDefault = getAutoIncrementDefault(databaseSystem, columnMapping, expectedColumn) + val autoIncrementDefault = + getAutoIncrementDefault(expectedColumn, columnMapping, databaseSystem, connectionMetadata) actualColumnDefault should be(autoIncrementDefault) } else { @@ -162,6 +165,8 @@ class LiquibaseComparisonSpec extends AnyFlatSpec with CromwellTimeoutSpec with } val actualForeignKey = actualForeignKeyOption getOrElse fail(s"Did not find $description") + actualForeignKey.getPrimaryKeyTable.getName should be(expectedForeignKey.getPrimaryKeyTable.getName) + actualForeignKey.getForeignKeyTable.getName should be(expectedForeignKey.getForeignKeyTable.getName) actualForeignKey.getPrimaryKeyColumns.asScala.map(ColumnDescription.from) should contain theSameElementsAs expectedForeignKey.getPrimaryKeyColumns.asScala.map(ColumnDescription.from) actualForeignKey.getForeignKeyColumns.asScala.map(ColumnDescription.from) should @@ -194,6 +199,7 @@ class LiquibaseComparisonSpec extends AnyFlatSpec with CromwellTimeoutSpec with val actualUniqueConstraint = actualUniqueConstraintOption getOrElse fail(s"Did not find $description") + actualUniqueConstraint.getRelation.getName should be(expectedUniqueConstraint.getRelation.getName) actualUniqueConstraint.getColumns.asScala.map(ColumnDescription.from) should contain theSameElementsAs expectedUniqueConstraint.getColumns.asScala.map(ColumnDescription.from) } @@ -228,7 +234,7 @@ class LiquibaseComparisonSpec extends AnyFlatSpec with CromwellTimeoutSpec with object LiquibaseComparisonSpec { private def get[T <: DatabaseObject : ClassTag : Ordering](databaseSnapshot: DatabaseSnapshot): Seq[T] = { val databaseObjectClass = classTag[T].runtimeClass.asInstanceOf[Class[T]] - databaseSnapshot.get(databaseObjectClass).asScala.toSeq.sorted + databaseSnapshot.get(databaseObjectClass).asScala.toSeq } private val DefaultNullBoolean = Boolean.box(false) @@ -276,8 +282,8 @@ object LiquibaseComparisonSpec { case class ColumnMapping ( - typeMapping: Map[ColumnType, ColumnType] = Map.empty, - defaultMapping: Map[ColumnDefault, ColumnDefault] = Map.empty, + typeMapping: PartialFunction[ColumnType, ColumnType] = PartialFunction.empty, + defaultMapping: Map[ColumnDefault, AnyRef] = Map.empty, ) /** Generate the expected PostgreSQL sequence name for a column. */ @@ -315,6 +321,9 @@ object LiquibaseComparisonSpec { private val HsqldbTypeInteger = ColumnType("INTEGER", Option(32)) private val HsqldbTypeTimestamp = ColumnType("TIMESTAMP") + // Defaults as they are represented in HSQLDB that will have different representations in other DBMS. + private val HsqldbDefaultBooleanTrue = ColumnDefault(HsqldbTypeBoolean, Boolean.box(true)) + // Nothing to map as the original is also HSQLDB private val HsqldbColumnMapping = ColumnMapping() @@ -331,13 +340,25 @@ object LiquibaseComparisonSpec { HsqldbTypeTimestamp -> ColumnType("DATETIME"), ), defaultMapping = Map( - ColumnDefault(HsqldbTypeBoolean, Boolean.box(true)) -> - ColumnDefault(ColumnType("TINYINT", Option(3)), Int.box(1)), + HsqldbDefaultBooleanTrue -> Int.box(1) ), ) - // MariaDB should behave exactly the same as MySQL - private val MariadbColumnMapping = MysqldbColumnMapping + // MariaDB should behave similar to MySQL except that only LOBs have sizes + private val MariadbColumnMapping = + ColumnMapping( + typeMapping = Map( + HsqldbTypeBigInt -> ColumnType("BIGINT"), + HsqldbTypeBlob -> ColumnType("LONGBLOB", Option(2147483647)), + HsqldbTypeBoolean -> ColumnType("TINYINT"), + HsqldbTypeClob -> ColumnType("LONGTEXT", Option(2147483647)), + HsqldbTypeInteger -> ColumnType("INT"), + HsqldbTypeTimestamp -> ColumnType("DATETIME"), + ), + defaultMapping = Map( + HsqldbDefaultBooleanTrue -> Int.box(1), + ), + ) private val PostgresqlColumnMapping = ColumnMapping( @@ -367,24 +388,26 @@ object LiquibaseComparisonSpec { */ private def getColumnType(column: Column, columnMapping: ColumnMapping): ColumnType = { val columnType = ColumnType.from(column) - columnMapping.typeMapping.getOrElse(columnType, columnType) + columnMapping.typeMapping.applyOrElse[ColumnType, ColumnType](columnType, _ => columnType) } /** * Returns the default for the column, either from ColumnMapping or the column itself. */ private def getColumnDefault(column: Column, columnMapping: ColumnMapping): AnyRef = { - columnMapping.defaultMapping get ColumnDefault.from(column) map (_.defaultValue) getOrElse column.getDefaultValue + columnMapping.defaultMapping.getOrElse(ColumnDefault.from(column), column.getDefaultValue) } /** * Return the default for the auto increment column. */ - private def getAutoIncrementDefault(databaseSystem: DatabaseSystem, + private def getAutoIncrementDefault(column: Column, columnMapping: ColumnMapping, - column: Column): ColumnDefault = { + databaseSystem: DatabaseSystem, + connectionMetadata: ConnectionMetadata, + ): ColumnDefault = { databaseSystem.platform match { - case PostgresqlDatabasePlatform => + case PostgresqlDatabasePlatform if connectionMetadata.databaseMajorVersion <= 9 => val columnType = column.getType.getTypeName match { case "BIGINT" => ColumnType("BIGSERIAL", None) case "INTEGER" => ColumnType("SERIAL", None) @@ -441,8 +464,8 @@ object LiquibaseComparisonSpec { /** * Returns an optional extra check to ensure that sequences have the same types as their auto increment columns. * - * This is because PostgreSQL requires two statements to modify SERIAL columns to BIGSERIAL, one to widen the column, - * and another to widen the sequence. + * This is because PostgreSQL <= 9 requires two statements to modify SERIAL columns to BIGSERIAL, one to widen the + * column, and another to widen the sequence. * * https://stackoverflow.com/questions/52195303/postgresql-primary-key-id-datatype-from-serial-to-bigserial#answer-52195920 * https://www.postgresql.org/docs/11/datatype-numeric.html#DATATYPE-SERIAL @@ -455,8 +478,6 @@ object LiquibaseComparisonSpec { case PostgresqlDatabasePlatform if column.isAutoIncrement && connectionMetadata.databaseMajorVersion <= 9 => // "this is currently always bigint" --> https://www.postgresql.org/docs/9.6/infoschema-sequences.html Option("bigint") - case PostgresqlDatabasePlatform if column.isAutoIncrement => - Option(column.getType.getTypeName.toLowerCase) case _ => None } } @@ -467,8 +488,8 @@ object LiquibaseComparisonSpec { import database.dataAccess.driver.api._ databaseSystem.platform match { case PostgresqlDatabasePlatform if column.isAutoIncrement => - //noinspection SqlDialectInspection - sql"""select data_type + //noinspection SqlDialectInspection + sql"""select data_type from INFORMATION_SCHEMA.sequences where sequence_name = '#${postgresqlSeqName(column)}' """.as[String].head diff --git a/services/src/test/scala/cromwell/services/database/LobSpec.scala b/services/src/test/scala/cromwell/services/database/LobSpec.scala index ed6723b90fc..a92cf84aed9 100644 --- a/services/src/test/scala/cromwell/services/database/LobSpec.scala +++ b/services/src/test/scala/cromwell/services/database/LobSpec.scala @@ -1,7 +1,5 @@ package cromwell.services.database -import java.time.OffsetDateTime - import com.dimafeng.testcontainers.Container import common.assertion.CromwellTimeoutSpec import cromwell.core.Tags._ @@ -9,19 +7,21 @@ import cromwell.core.WorkflowId import cromwell.database.sql.SqlConverters._ import cromwell.database.sql.joins.JobStoreJoin import cromwell.database.sql.tables.{JobStoreEntry, JobStoreSimpletonEntry, WorkflowStoreEntry} -import javax.sql.rowset.serial.SerialBlob import org.scalatest.concurrent.ScalaFutures import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers +import java.time.OffsetDateTime +import javax.sql.rowset.serial.SerialBlob import scala.concurrent.ExecutionContext import scala.concurrent.duration._ class LobSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers with ScalaFutures { - implicit val executionContext = ExecutionContext.global + implicit val executionContext: ExecutionContext = ExecutionContext.global - implicit val defaultPatience = PatienceConfig(timeout = scaled(5.seconds), interval = scaled(100.millis)) + implicit val defaultPatience: PatienceConfig = + PatienceConfig(timeout = scaled(5.seconds), interval = scaled(100.millis)) DatabaseSystem.All foreach { databaseSystem => @@ -35,7 +35,7 @@ class LobSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers with Sc containerOpt.foreach { _.start } } - it should "fail to store and retrieve empty blobs" taggedAs DbmsTest in { + it should "store empty blobs" taggedAs DbmsTest in { // See notes in BytesToBlobOption import eu.timepit.refined.auto._ val clob = "".toClob(default = "{}") diff --git a/services/src/test/scala/cromwell/services/keyvalue/impl/KeyValueDatabaseSpec.scala b/services/src/test/scala/cromwell/services/keyvalue/impl/KeyValueDatabaseSpec.scala index 16223fdaaf0..813079480dc 100644 --- a/services/src/test/scala/cromwell/services/keyvalue/impl/KeyValueDatabaseSpec.scala +++ b/services/src/test/scala/cromwell/services/keyvalue/impl/KeyValueDatabaseSpec.scala @@ -125,7 +125,8 @@ object KeyValueDatabaseSpec { private def getFailureRegex(databaseSystem: DatabaseSystem): String = { databaseSystem.platform match { case HsqldbDatabasePlatform => - "integrity constraint violation: NOT NULL check constraint; SYS_CT_10591 table: JOB_KEY_VALUE_ENTRY column: STORE_VALUE" + """integrity constraint violation: NOT NULL check constraint; """ + + """SYS_CT_\d+ table: JOB_KEY_VALUE_ENTRY column: STORE_VALUE""" case MariadbDatabasePlatform => """\(conn=\d+\) Column 'STORE_VALUE' cannot be null""" case MysqlDatabasePlatform => "Column 'STORE_VALUE' cannot be null" case PostgresqlDatabasePlatform => diff --git a/src/ci/bin/test.inc.sh b/src/ci/bin/test.inc.sh index dd03e6dd01b..39351570f6c 100644 --- a/src/ci/bin/test.inc.sh +++ b/src/ci/bin/test.inc.sh @@ -1085,6 +1085,10 @@ cromwell::private::login_docker() { cromwell::private::render_secure_resources() { # Copy the CI resources, then render the secure resources using Vault + + # Cosmetic: run this first so that renderCiResources doesn't write the docker pull output to SBT's standard error + docker pull broadinstitute/dsde-toolbox:dev + sbt -Dsbt.supershell=false --warn renderCiResources \ || if [[ "${CROMWELL_BUILD_IS_CI}" == "true" ]]; then echo From ad20b9a79505f78cb817ef731b7d42d400021e8f Mon Sep 17 00:00:00 2001 From: Khalid Shakir Date: Sat, 26 Jun 2021 09:28:10 -0400 Subject: [PATCH 2/2] PR Fixup: Remove cosmetic patch added in another PR --- src/ci/bin/test.inc.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/ci/bin/test.inc.sh b/src/ci/bin/test.inc.sh index 88f1eb2fc8e..fb2a695632e 100644 --- a/src/ci/bin/test.inc.sh +++ b/src/ci/bin/test.inc.sh @@ -1087,10 +1087,6 @@ cromwell::private::render_secure_resources() { # Avoid docker output to sbt's stderr by pulling the image here docker pull broadinstitute/dsde-toolbox:dev | cat # Copy the CI resources, then render the secure resources using Vault - - # Cosmetic: run this first so that renderCiResources doesn't write the docker pull output to SBT's standard error - docker pull broadinstitute/dsde-toolbox:dev - sbt -Dsbt.supershell=false --warn renderCiResources \ || if [[ "${CROMWELL_BUILD_IS_CI}" == "true" ]]; then echo