Chore: upgrade postgres version on tests and local docker to 11.x (#664)

* Upgrade postgres version on tests and local docker

 * fix db tests with ordering based on encoding of strings
 * clean up the testcontainers setup

* Guarantee collation uniform handling in tests and local docker postgres

 * explicitly set (when possible) and verify the sorting of strings
 * avoid the alpine db image since it doesn't correctly use the
   en_US collation type

docker-compose.yml

@@ -2,14 +2,14 @@ version: "3.1"
 
 services:
   conseil-postgres:
-    image: postgres:10.3-alpine
+    image: postgres:11.6
     ports:
     - 5432:5432
     environment:
       POSTGRES_USER: "conseiluser"
       POSTGRES_PASSWORD: "p@ssw0rd"
       POSTGRES_DB: "conseil-local"
-      POSTGRES_INITDB_ARGS: "--nosync --lc-collate=C"
+      POSTGRES_INITDB_ARGS: "--lc-collate=en_US.UTF-8 -E UTF8"
     volumes:
     - "./pgdata:/var/lib/postgresql/data"
     - "./sql/conseil.sql:/docker-entrypoint-initdb.d/conseil.sql"

src/test/resources/in-memory-db/init-script.sql

@@ -0,0 +1,3 @@
+
+SET client_encoding = 'UTF8';
+CREATE SCHEMA tezos;

src/test/scala/tech/cryptonomic/conseil/tezos/InMemoryDatabase.scala

@@ -12,18 +12,22 @@ import scala.concurrent.duration._
   */
 trait InMemoryDatabase extends BeforeAndAfterAll with BeforeAndAfterEach {
   self: TestSuite =>
-  import java.nio.file._
   import slick.jdbc.PostgresProfile.api._
 
-  val dbInstance = new PostgreSQLContainer()
+  //#JavaThankYou
+  val dbInstance =
+    new PostgreSQLContainer("postgres:11.6")
+      .asInstanceOf[PostgreSQLContainer[_]]
+      .withInitScript("in-memory-db/init-script.sql") //startup will prepare the schema
+      .asInstanceOf[PostgreSQLContainer[_]]
+      .withCommand("-c full_page_writes=off") //should improve performance for the tests
+      .asInstanceOf[PostgreSQLContainer[_]]
+
   dbInstance.start()
 
   /** how to name the database schema for the test */
   protected val databaseName = dbInstance.getDatabaseName
 
-  /** here are temp files for the embedded process, can wipe out if needed */
-  protected val cachedRuntimePath = Paths.get("test-postgres-path")
-
   /** defines configuration for a randomly named embedded instance */
   protected lazy val confString =
     s"""testdb = {
@@ -57,7 +61,6 @@ trait InMemoryDatabase extends BeforeAndAfterAll with BeforeAndAfterEach {
   )
 
   protected val dbSchema = Tables.schema
-  allTables.map(_.schema).reduce(_ ++ _)
 
   /**
     * calling deletes manually is needed to obviate the fact
@@ -71,11 +74,6 @@ trait InMemoryDatabase extends BeforeAndAfterAll with BeforeAndAfterEach {
   override protected def beforeAll(): Unit = {
     super.beforeAll()
     dbInstance.start()
-    Await.result(dbHandler.run(sql"CREATE SCHEMA IF NOT EXISTS tezos".as[Int]), 1.second)
-    Await.result(
-      dbHandler.run(sql"""ALTER DATABASE "#$databaseName" SET search_path TO tezos,public""".as[Int]),
-      1.second
-    )
     Await.result(dbHandler.run(dbSchema.create), 1.second)
   }
 
@@ -87,8 +85,9 @@ trait InMemoryDatabase extends BeforeAndAfterAll with BeforeAndAfterEach {
   }
 
   override protected def beforeEach(): Unit = {
-    Await.ready(dbHandler.run(truncateAll), 1.second)
     super.beforeEach()
+    Await.ready(dbHandler.run(truncateAll), 1.second)
+    ()
   }
 
 }

src/test/scala/tech/cryptonomic/conseil/tezos/TezosDatabaseOperationsTest.scala

@@ -43,6 +43,12 @@ class TezosDatabaseOperationsTest
       val sut = TezosDatabaseOperations
       val feesToConsider = 1000
 
+      "use the right collation" in {
+        val ordered =
+          dbHandler.run(sql"SELECT val FROM unnest(ARRAY['a', 'b', 'A', 'B']) val ORDER BY val".as[String]).futureValue
+        ordered should contain inOrderOnly ("a", "A", "b", "B")
+      }
+
       "write fees" in {
         implicit val randomSeed = RandomSeed(testReferenceTimestamp.getTime)
 
@@ -2254,7 +2260,8 @@ class TezosDatabaseOperationsTest
 
         import tech.cryptonomic.conseil.util.DatabaseUtil.QueryBuilder._
         val columns = List(SimpleField("level"), SimpleField("proto"), SimpleField("protocol"), SimpleField("hash"))
-        val tableName = Tables.Blocks.baseTableRow.tableName
+        val tableSpace = "tezos"
+        val tableName = s"$tableSpace.${Tables.Blocks.baseTableRow.tableName}"
         val populateAndTest = for {
           _ <- Tables.Blocks ++= blocksTmp
           generatedQuery <- makeQuery(tableName, columns, List.empty).as[AnyMap]