From 46be283114dbb796a12418459c28c9733c3345f7 Mon Sep 17 00:00:00 2001 From: Ivano Pagano Date: Mon, 6 Jan 2020 16:39:14 +0100 Subject: [PATCH] Chore: upgrade postgres version on tests and local docker to 11.x (#664) * Upgrade postgres version on tests and local docker * fix db tests with ordering based on encoding of strings * clean up the testcontainers setup * Guarantee collation uniform handling in tests and local docker postgres * explicitly set (when possible) and verify the sorting of strings * avoid the alpine db image since it doesn't correctly use the en_US collation type --- docker-compose.yml | 4 ++-- .../resources/in-memory-db/init-script.sql | 3 +++ .../conseil/tezos/InMemoryDatabase.scala | 23 +++++++++---------- .../tezos/TezosDatabaseOperationsTest.scala | 9 +++++++- 4 files changed, 24 insertions(+), 15 deletions(-) create mode 100644 src/test/resources/in-memory-db/init-script.sql diff --git a/docker-compose.yml b/docker-compose.yml index 2692001f7..2e31d25cc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,14 +2,14 @@ version: "3.1" services: conseil-postgres: - image: postgres:10.3-alpine + image: postgres:11.6 ports: - 5432:5432 environment: POSTGRES_USER: "conseiluser" POSTGRES_PASSWORD: "p@ssw0rd" POSTGRES_DB: "conseil-local" - POSTGRES_INITDB_ARGS: "--nosync --lc-collate=C" + POSTGRES_INITDB_ARGS: "--lc-collate=en_US.UTF-8 -E UTF8" volumes: - "./pgdata:/var/lib/postgresql/data" - "./sql/conseil.sql:/docker-entrypoint-initdb.d/conseil.sql" diff --git a/src/test/resources/in-memory-db/init-script.sql b/src/test/resources/in-memory-db/init-script.sql new file mode 100644 index 000000000..bb0c6646d --- /dev/null +++ b/src/test/resources/in-memory-db/init-script.sql @@ -0,0 +1,3 @@ + +SET client_encoding = 'UTF8'; +CREATE SCHEMA tezos; \ No newline at end of file diff --git a/src/test/scala/tech/cryptonomic/conseil/tezos/InMemoryDatabase.scala b/src/test/scala/tech/cryptonomic/conseil/tezos/InMemoryDatabase.scala index cb19c9164..5e4045a96 100644 --- a/src/test/scala/tech/cryptonomic/conseil/tezos/InMemoryDatabase.scala +++ b/src/test/scala/tech/cryptonomic/conseil/tezos/InMemoryDatabase.scala @@ -12,18 +12,22 @@ import scala.concurrent.duration._ */ trait InMemoryDatabase extends BeforeAndAfterAll with BeforeAndAfterEach { self: TestSuite => - import java.nio.file._ import slick.jdbc.PostgresProfile.api._ - val dbInstance = new PostgreSQLContainer() + //#JavaThankYou + val dbInstance = + new PostgreSQLContainer("postgres:11.6") + .asInstanceOf[PostgreSQLContainer[_]] + .withInitScript("in-memory-db/init-script.sql") //startup will prepare the schema + .asInstanceOf[PostgreSQLContainer[_]] + .withCommand("-c full_page_writes=off") //should improve performance for the tests + .asInstanceOf[PostgreSQLContainer[_]] + dbInstance.start() /** how to name the database schema for the test */ protected val databaseName = dbInstance.getDatabaseName - /** here are temp files for the embedded process, can wipe out if needed */ - protected val cachedRuntimePath = Paths.get("test-postgres-path") - /** defines configuration for a randomly named embedded instance */ protected lazy val confString = s"""testdb = { @@ -57,7 +61,6 @@ trait InMemoryDatabase extends BeforeAndAfterAll with BeforeAndAfterEach { ) protected val dbSchema = Tables.schema - allTables.map(_.schema).reduce(_ ++ _) /** * calling deletes manually is needed to obviate the fact @@ -71,11 +74,6 @@ trait InMemoryDatabase extends BeforeAndAfterAll with BeforeAndAfterEach { override protected def beforeAll(): Unit = { super.beforeAll() dbInstance.start() - Await.result(dbHandler.run(sql"CREATE SCHEMA IF NOT EXISTS tezos".as[Int]), 1.second) - Await.result( - dbHandler.run(sql"""ALTER DATABASE "#$databaseName" SET search_path TO tezos,public""".as[Int]), - 1.second - ) Await.result(dbHandler.run(dbSchema.create), 1.second) } @@ -87,8 +85,9 @@ trait InMemoryDatabase extends BeforeAndAfterAll with BeforeAndAfterEach { } override protected def beforeEach(): Unit = { - Await.ready(dbHandler.run(truncateAll), 1.second) super.beforeEach() + Await.ready(dbHandler.run(truncateAll), 1.second) + () } } diff --git a/src/test/scala/tech/cryptonomic/conseil/tezos/TezosDatabaseOperationsTest.scala b/src/test/scala/tech/cryptonomic/conseil/tezos/TezosDatabaseOperationsTest.scala index 70c0298d7..e597738af 100644 --- a/src/test/scala/tech/cryptonomic/conseil/tezos/TezosDatabaseOperationsTest.scala +++ b/src/test/scala/tech/cryptonomic/conseil/tezos/TezosDatabaseOperationsTest.scala @@ -43,6 +43,12 @@ class TezosDatabaseOperationsTest val sut = TezosDatabaseOperations val feesToConsider = 1000 + "use the right collation" in { + val ordered = + dbHandler.run(sql"SELECT val FROM unnest(ARRAY['a', 'b', 'A', 'B']) val ORDER BY val".as[String]).futureValue + ordered should contain inOrderOnly ("a", "A", "b", "B") + } + "write fees" in { implicit val randomSeed = RandomSeed(testReferenceTimestamp.getTime) @@ -2254,7 +2260,8 @@ class TezosDatabaseOperationsTest import tech.cryptonomic.conseil.util.DatabaseUtil.QueryBuilder._ val columns = List(SimpleField("level"), SimpleField("proto"), SimpleField("protocol"), SimpleField("hash")) - val tableName = Tables.Blocks.baseTableRow.tableName + val tableSpace = "tezos" + val tableName = s"$tableSpace.${Tables.Blocks.baseTableRow.tableName}" val populateAndTest = for { _ <- Tables.Blocks ++= blocksTmp generatedQuery <- makeQuery(tableName, columns, List.empty).as[AnyMap]