1023 - Remove db redundancy in config file (#1025)

* WiP * small updates to config and docker-compose.yml
Cryptonomic · Aug 17, 2021 · 0920177 · 0920177
1 parent e6366eb
commit 0920177
Show file tree

Hide file tree

Showing 10 changed files with 59 additions and 93 deletions.
diff --git a/conseil-api/src/main/resources/application.conf b/conseil-api/src/main/resources/application.conf
@@ -12,29 +12,6 @@ conseil {
   high-cardinality-limit: 100
   startup-deadline: 5 minutes
 
-  # Replace database name, user and password in an env-specific config file.
-  db {
-    dataSourceClass: "org.postgresql.ds.PGSimpleDataSource"
-
-    properties {
-      user: "foo"
-      user: ${?CONSEIL_API_DB_USER}
-
-      password: "bar"
-      password: ${?CONSEIL_API_DB_PASSWORD}
-
-      url: "jdbc:postgresql://localhost:5432/postgres"
-      url: ${?CONSEIL_API_DB_URL}
-
-      reWriteBatchedInserts: true
-    }
-    # The following numbers are based on literature from here: https://github.com/brettwooldridge/HikariCP/wiki/About-Pool-Sizing
-    # We might want to fine-tune these on the actual infrastructure, doing testing with different values
-    # Please keep both values aligned in your configuration to avoid this issue: https://github.com/dnvriend/akka-persistence-jdbc/issues/177
-    numThreads: 10
-    maxConnections: 10
-  }
-
   # Security settings
   security.api-keys {
     # List of authorized API keys needed to query Conseil

diff --git a/conseil-common/src/main/resources/reference.conf b/conseil-common/src/main/resources/reference.conf
@@ -24,10 +24,10 @@ platforms: [
   {
     name: "tezos"
 
-    network: "zeronet"
+    network: "mainnet"
     network: ${?CONSEIL_XTZ_NETWORK}
 
-    enabled: false
+    enabled: true
     enabled: ${?CONSEIL_XTZ_ENABLED}
 
     node: {
@@ -40,21 +40,21 @@ platforms: [
       port: 8732
       port: ${?CONSEIL_XTZ_NODE_PORT}
 
-      path-prefix: "tezos/zeronet/"
+      path-prefix: ""
       path-prefix: ${?CONSEIL_XTZ_NODE_PATH_PREFIX}
     }
 
     db {
       dataSourceClass: "org.postgresql.ds.PGSimpleDataSource"
 
       properties {
-        user: "foo"
+        user: "conseiluser"
         user: ${?CONSEIL_XTZ_DB_USER}
 
-        password: "bar"
+        password: "p@ssw0rd"
         password: ${?CONSEIL_XTZ_DB_PASSWORD}
 
-        url: "jdbc:postgresql://localhost:5432/postgres"
+        url: "jdbc:postgresql://localhost:5432/conseil-local"
         url: ${?CONSEIL_XTZ_DB_URL}
 
         reWriteBatchedInserts: true
@@ -76,7 +76,7 @@ platforms: [
     enabled: ${?CONSEIL_BTC_ENABLED}
 
     node: {
-      protocol: "https"
+      protocol: "http"
       protocol: ${?CONSEIL_BTC_NODE_PROTOCOL}
 
       hostname: "localhost"
@@ -97,14 +97,14 @@ platforms: [
       dataSourceClass: "org.postgresql.ds.PGSimpleDataSource"
 
       properties {
-        user: "foo"
-        user: ${?CONSEIL_BTC_DB_USER}
+        user: "conseiluser"
+        user: ${?CONSEIL_XTZ_DB_USER}
 
-        password: "bar"
-        password: ${?CONSEIL_BTC_DB_PASSWORD}
+        password: "p@ssw0rd"
+        password: ${?CONSEIL_XTZ_DB_PASSWORD}
 
-        url: "jdbc:postgresql://localhost:5432/postgres"
-        url: ${?CONSEIL_BTC_DB_URL}
+        url: "jdbc:postgresql://localhost:5432/conseil-local"
+        url: ${?CONSEIL_XTZ_DB_URL}
 
         reWriteBatchedInserts: true
       }
@@ -148,14 +148,14 @@ platforms: [
       dataSourceClass: "org.postgresql.ds.PGSimpleDataSource"
 
       properties {
-        user: "foo"
-        user: ${?CONSEIL_ETH_DB_USER}
+        user: "conseiluser"
+        user: ${?CONSEIL_XTZ_DB_USER}
 
-        password: "bar"
-        password: ${?CONSEIL_ETH_DB_PASSWORD}
+        password: "p@ssw0rd"
+        password: ${?CONSEIL_XTZ_DB_PASSWORD}
 
-        url: "jdbc:postgresql://localhost:5432/postgres"
-        url: ${?CONSEIL_ETH_DB_URL}
+        url: "jdbc:postgresql://localhost:5432/conseil-local"
+        url: ${?CONSEIL_XTZ_DB_URL}
 
         reWriteBatchedInserts: true
       }

diff --git a/conseil-common/src/main/scala/tech/cryptonomic/conseil/common/util/DatabaseUtil.scala b/conseil-common/src/main/scala/tech/cryptonomic/conseil/common/util/DatabaseUtil.scala
@@ -11,8 +11,6 @@ import tech.cryptonomic.conseil.common.generic.chain.DataTypes._
   * Utility functions and members for common database operations.
   */
 object DatabaseUtil {
-  lazy val conseilDb = Database.forConfig("conseil.db")
-  lazy val lorreDb = Database.forConfig("lorre.db")
 
   /**
     * Utility object for generic query composition with SQL interpolation

diff --git a/conseil-lorre/src/main/resources/application.conf b/conseil-lorre/src/main/resources/application.conf
@@ -42,29 +42,6 @@ lorre {
     fork-handling-is-on: ${?CONSEIL_LORRE_FORK_DETECTION_ENABLED}
   }
 
-  # Replace database name, user and password in an env-specific config file.
-  db {
-    dataSourceClass: "org.postgresql.ds.PGSimpleDataSource"
-
-    properties {
-      user: "foo"
-      user: ${?CONSEIL_LORRE_DB_USER}
-
-      password: "bar"
-      password: ${?CONSEIL_LORRE_DB_PASSWORD}
-
-      url: "jdbc:postgresql://localhost:5432/postgres"
-      url: ${?CONSEIL_LORRE_DB_URL}
-
-      reWriteBatchedInserts: true
-    }
-    # The following numbers are based on literature from here: https://github.com/brettwooldridge/HikariCP/wiki/About-Pool-Sizing
-    # We might want to fine-tune these on the actual infrastructure, doing testing with different values
-    # Please keep both values aligned in your configuration to avoid this issue: https://github.com/dnvriend/akka-persistence-jdbc/issues/177
-    numThreads: 10
-    maxConnections: 10
-  }
-
   # TODO Should we wrap configuration from below into 'Tezos' secion? The same in the *Configuration class?
 
   batched-fetches {

diff --git a/conseil-lorre/src/main/scala/tech/cryptonomic/conseil/indexer/Lorre.scala b/conseil-lorre/src/main/scala/tech/cryptonomic/conseil/indexer/Lorre.scala
@@ -1,5 +1,6 @@
 package tech.cryptonomic.conseil.indexer
 
+import slick.jdbc.JdbcBackend.Database
 import tech.cryptonomic.conseil.common.config.Platforms.{BitcoinConfiguration, TezosConfiguration}
 import tech.cryptonomic.conseil.indexer.config.LorreAppConfig.LORRE_FAILURE_IGNORE_VAR
 import tech.cryptonomic.conseil.indexer.config.LorreAppConfig
@@ -47,16 +48,20 @@ object Lorre extends App with LorreAppConfig with LorreInfoLogging {
   val indexer = platformConf match {
     case conf: QuorumConfiguration =>
       logger.info("Initializing indexer for Quorum Blockchain.")
-      EthereumIndexer.fromConfig(lorreConf, conf.toEthereumConfiguration)
+      val db = Database.forConfig("", conf.db)
+      EthereumIndexer.fromConfig(lorreConf, conf.toEthereumConfiguration, db)
     case conf: EthereumConfiguration =>
       logger.info("Initializing indexer for Ethereum Blockchain.")
-      EthereumIndexer.fromConfig(lorreConf, conf)
+      val db = Database.forConfig("", conf.db)
+      EthereumIndexer.fromConfig(lorreConf, conf, db)
     case conf: TezosConfiguration =>
       logger.info("Initializing indexer for Tezos Blockchain.")
-      TezosIndexer.fromConfig(lorreConf, conf, callsConf, streamingClientConf, batchingConf)
+      val db = Database.forConfig("", conf.db)
+      TezosIndexer.fromConfig(lorreConf, conf, callsConf, streamingClientConf, batchingConf, db)
     case conf: BitcoinConfiguration =>
       logger.info("Initializing indexer for Bitcoin Blockchain.")
-      BitcoinIndexer.fromConfig(lorreConf, conf)
+      val db = Database.forConfig("", conf.db)
+      BitcoinIndexer.fromConfig(lorreConf, conf, db)
   }
 
   try {

diff --git a/conseil-lorre/src/main/scala/tech/cryptonomic/conseil/indexer/bitcoin/BitcoinIndexer.scala b/conseil-lorre/src/main/scala/tech/cryptonomic/conseil/indexer/bitcoin/BitcoinIndexer.scala
@@ -30,7 +30,8 @@ import tech.cryptonomic.conseil.common.rpc.RpcClient
   */
 class BitcoinIndexer(
     lorreConf: LorreConfiguration,
-    bitcoinConf: BitcoinConfiguration
+    bitcoinConf: BitcoinConfiguration,
+    db: Database
 ) extends LorreIndexer
     with LorreProgressLogging
     with ConseilLogSupport {
@@ -124,7 +125,7 @@ class BitcoinIndexer(
       )
 
       tx <- Transactor
-        .fromDatabase[IO](IO.delay(DatabaseUtil.lorreDb))
+        .fromDatabase[IO](IO.delay(db))
         .map(_.configure(transactorConfig.transactionally)) // run operations in transaction
 
       bitcoinOperations <- BitcoinOperations.resource(rpcClient, tx, bitcoinConf.batching)
@@ -141,7 +142,8 @@ object BitcoinIndexer {
     */
   def fromConfig(
       lorreConf: LorreConfiguration,
-      bitcoinConf: BitcoinConfiguration
+      bitcoinConf: BitcoinConfiguration,
+      db: Database
   ): LorreIndexer =
-    new BitcoinIndexer(lorreConf, bitcoinConf)
+    new BitcoinIndexer(lorreConf, bitcoinConf, db)
 }
diff --git a/conseil-lorre/src/main/scala/tech/cryptonomic/conseil/indexer/ethereum/EthereumIndexer.scala b/conseil-lorre/src/main/scala/tech/cryptonomic/conseil/indexer/ethereum/EthereumIndexer.scala
@@ -28,7 +28,8 @@ import tech.cryptonomic.conseil.indexer.logging.LorreProgressLogging
   */
 class EthereumIndexer(
     lorreConf: LorreConfiguration,
-    ethereumConf: EthereumConfiguration
+    ethereumConf: EthereumConfiguration,
+    db: Database
 ) extends LorreIndexer
     with LorreProgressLogging {
 
@@ -120,7 +121,7 @@ class EthereumIndexer(
       )
 
       tx <- Transactor
-        .fromDatabase[IO](IO.delay(DatabaseUtil.lorreDb))
+        .fromDatabase[IO](IO.delay(db))
         .map(_.configure(transactorConfig.transactionally)) // run operations in transaction
 
       ethereumOperations <- EthereumOperations.resource(rpcClient, tx, ethereumConf.batching)
@@ -137,7 +138,8 @@ object EthereumIndexer {
     */
   def fromConfig(
       lorreConf: LorreConfiguration,
-      ethereumConf: EthereumConfiguration
+      ethereumConf: EthereumConfiguration,
+    db: Database
   ): LorreIndexer =
-    new EthereumIndexer(lorreConf, ethereumConf)
+    new EthereumIndexer(lorreConf, ethereumConf, db: Database)
 }
diff --git a/conseil-lorre/src/main/scala/tech/cryptonomic/conseil/indexer/tezos/TezosFeeOperations.scala b/conseil-lorre/src/main/scala/tech/cryptonomic/conseil/indexer/tezos/TezosFeeOperations.scala
@@ -1,5 +1,6 @@
 package tech.cryptonomic.conseil.indexer.tezos
 
+import slick.jdbc.JdbcBackend.Database
 import tech.cryptonomic.conseil.common.io.Logging.ConseilLogSupport
 import tech.cryptonomic.conseil.indexer.tezos.{TezosDatabaseOperations => TezosDb}
 
@@ -10,10 +11,9 @@ import scala.concurrent.duration.FiniteDuration
 /**
   * Helper classes and functions used for average fee calculations.
   */
-private[tezos] object TezosFeeOperations extends ConseilLogSupport {
+private[tezos] class TezosFeeOperations(db: Database) extends ConseilLogSupport {
   // We are getting access to the same database instance, but it should not be a shared object in a long-run.
   // After the splitting we should refactor this part.
-  import tech.cryptonomic.conseil.common.util.DatabaseUtil.{lorreDb => db}
 
   private val operationKinds = List(
     "seed_nonce_revelation",

diff --git a/conseil-lorre/src/main/scala/tech/cryptonomic/conseil/indexer/tezos/TezosIndexer.scala b/conseil-lorre/src/main/scala/tech/cryptonomic/conseil/indexer/tezos/TezosIndexer.scala
@@ -7,6 +7,7 @@ import akka.stream.scaladsl.Source
 import mouse.any._
 import cats.instances.future._
 import cats.syntax.applicative._
+import slick.jdbc.JdbcBackend.Database
 import tech.cryptonomic.conseil.common.config.Platforms.{BlockchainPlatform, TezosConfiguration}
 import tech.cryptonomic.conseil.common.config._
 import tech.cryptonomic.conseil.common.io.Logging.ConseilLogSupport
@@ -56,6 +57,7 @@ class TezosIndexer private (
     metadataProcessor: MetadataProcessor,
     accountsResetHandler: AccountsResetHandler,
     forkHandler: ForkHandler[Future, TezosBlockHash],
+    feeOperations: TezosFeeOperations,
     terminationSequence: () => Future[ShutdownComplete]
 )(
     implicit
@@ -110,7 +112,7 @@ class TezosIndexer private (
         reloadedAccountEvents.getOrElse(accountResetEvents)
       )
       _ <- processTezosBlocks(maxLevel)
-      _ <- TezosFeeOperations
+      _ <- feeOperations
         .processTezosAverageFees(lorreConf.feesAverageTimeWindow)
         .whenA(iteration % lorreConf.feeUpdateInterval == 0)
       _ <- rightsProcessor.updateRightsTimestamps()
@@ -268,7 +270,8 @@ object TezosIndexer extends ConseilLogSupport {
       conf: TezosConfiguration,
       callsConf: NetworkCallsConfiguration,
       streamingClientConf: HttpStreamingConfiguration,
-      batchingConf: BatchFetchConfiguration
+      batchingConf: BatchFetchConfiguration,
+      db: Database
   ): LorreIndexer = {
     val selectedNetwork = conf.network
 
@@ -281,7 +284,6 @@ object TezosIndexer extends ConseilLogSupport {
       ignoreProcessFailuresOrigin.exists(ignore => ignore == "true" || ignore == "yes")
 
     /* Here we collect all internal service operations and resources, needed to run the indexer */
-    val db = DatabaseUtil.lorreDb
     val indexedData = new TezosIndexedDataOperations(db)
 
     /* collects data from the remote tezos node */
@@ -423,6 +425,8 @@ object TezosIndexer extends ConseilLogSupport {
         amender = TezosForkInvalidatingAmender(db)
       )
 
+    val feeOperations = new TezosFeeOperations(db)
+
     /* the shutdown sequence to free resources */
     val gracefulTermination = () =>
       for {
@@ -444,6 +448,7 @@ object TezosIndexer extends ConseilLogSupport {
       metadataProcessor,
       accountsResetHandler,
       forkHandler,
+      feeOperations,
       gracefulTermination
     )
   }

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -1,15 +1,15 @@
 version: "3.1"
 
 services:
-  # Bitcoin node for development, if you want to start your own node,
-  # uncomment below lines (note: this code starts archive node and it will sync for a long time, 
-  # we recommend to set up the Bitcoin node on a dedicated machine outside the Docker):
-  # bitcoin:
-  #   image: ruimarinho/bitcoin-core
-  #   command: "-txindex=1 -printtoconsole -rpcallowip=0.0.0.0/0 -rpcbind=0.0.0.0 -rpcauth=bitcoin:68d381f566193c1938b7b1f72288bebf$$6c2d9f021bf4dd5e7e896634f531d35bb5aa9d90c77686af2f79a312e9ff4b75"
-  #   ports:
-  #     - 8332:8332
-  #     - 18443:18443
+# Bitcoin node for development, if you want to start your own node,
+# uncomment below lines (note: this code starts archive node and it will sync for a long time,
+# we recommend to set up the Bitcoin node on a dedicated machine outside the Docker):
+#  bitcoin:
+#    image: ruimarinho/bitcoin-core
+#    command: "-txindex=1 -printtoconsole -rpcallowip=0.0.0.0/0 -rpcbind=0.0.0.0 -rpcauth=bitcoin:68d381f566193c1938b7b1f72288bebf$$6c2d9f021bf4dd5e7e896634f531d35bb5aa9d90c77686af2f79a312e9ff4b75"
+#    ports:
+#       - 8332:8332
+#       - 18443:18443
 
   conseil-postgres:
     image: postgres:11.9