Skip to content

Commit

Permalink
Merge pull request #620 from Cryptonomic/feature/597-Lorre-performanc…
Browse files Browse the repository at this point in the history 
…e-improvements

597 - Lorre performance improvements
  • Loading branch information
@vishakh
vishakh authored Nov 18, 2019
2 parents 644815c + b9303d6 commit eb6df16
Show file tree
Hide file tree
Showing 5 changed files with 57 additions and 86 deletions.
9 changes: 0 additions & 9 deletions sql/conseil.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -603,15 +603,6 @@ ALTER TABLE ONLY tezos.operation_groups
ADD CONSTRAINT block FOREIGN KEY (block_id) REFERENCES tezos.blocks(hash);


--
-- TOC entry 2117 (class 2606 OID 99726)
-- Name: accounts_checkpoint checkpoint_block_id_fkey; Type: FK CONSTRAINT; Schema: tezos; Owner: -
--

ALTER TABLE ONLY tezos.accounts_checkpoint
ADD CONSTRAINT checkpoint_block_id_fkey FOREIGN KEY (block_id) REFERENCES tezos.blocks(hash);


--
-- TOC entry 2119 (class 2606 OID 99741)
-- Name: delegates_checkpoint delegate_checkpoint_block_id_fkey; Type: FK CONSTRAINT; Schema: tezos; Owner: -
Expand Down
19 changes: 4 additions & 15 deletions src/main/scala/tech/cryptonomic/conseil/Lorre.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -272,27 +272,16 @@ object Lorre extends App with TezosErrors with LazyLogging with LorreAppConfig w
* Fetches voting data for the blocks and stores any relevant
* result into the appropriate database table
*/
private[this] def processVotesForBlocks(blocks: List[TezosTypes.Block]): Future[Option[Int]] = {
import cats.syntax.traverse._
import cats.syntax.foldable._
import cats.instances.list._
import cats.instances.option._
import cats.instances.int._
import slickeffect.implicits._
import slick.jdbc.PostgresProfile.api._

private[this] def processVotesForBlocks(blocks: List[TezosTypes.Block]): Future[Option[Int]] =
tezosNodeOperator.getVotingDetails(blocks).flatMap {
case (proposals, bakersBlocks, ballotsBlocks) =>
//this is a nested list, each block with many baker rolls
val writeBakers = bakersBlocks.traverse {
case (block, bakersRolls) => TezosDb.writeVotingRolls(bakersRolls, block)
}
val writeBakers = TezosDb.writeVotingRolls(bakersBlocks)

val combinedVoteWrites = writeBakers.map(_.combineAll.map(_ + proposals.size + ballotsBlocks.size))
val combinedVoteWrites = writeBakers.map(_.map(_ + proposals.size + ballotsBlocks.size))

db.run(combinedVoteWrites.transactionally)
db.run(combinedVoteWrites)
}
}

/* Fetches accounts from account-id and saves those associated with the latest operations
* (i.e.the highest block level)
Expand Down
107 changes: 49 additions & 58 deletions src/main/scala/tech/cryptonomic/conseil/tezos/Tables.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -220,13 +220,6 @@ trait Tables {
/** Database column asof SqlType(timestamptz) */
val asof: Rep[java.sql.Timestamp] = column[java.sql.Timestamp]("asof")

/** Foreign key referencing Blocks (database name checkpoint_block_id_fkey) */
lazy val blocksFk = foreignKey("checkpoint_block_id_fkey", blockId, Blocks)(
r => r.hash,
onUpdate = ForeignKeyAction.NoAction,
onDelete = ForeignKeyAction.NoAction
)

/** Index over (accountId) (database name ix_accounts_checkpoint_account_id) */
val index1 = index("ix_accounts_checkpoint_account_id", accountId)

Expand All @@ -249,7 +242,7 @@ trait Tables {
* @param spendable Database column spendable SqlType(bool), Default(None)
* @param delegateSetable Database column delegate_setable SqlType(bool), Default(None)
* @param delegateValue Database column delegate_value SqlType(varchar), Default(None)
* @param asof Database column asof SqlType(timestamptz) */
* @param asof Database column asof SqlType(timestamp) */
case class AccountsHistoryRow(
accountId: String,
blockId: String,
Expand Down Expand Up @@ -373,7 +366,7 @@ trait Tables {
/** Database column delegate_value SqlType(varchar), Default(None) */
val delegateValue: Rep[Option[String]] = column[Option[String]]("delegate_value", O.Default(None))

/** Database column asof SqlType(timestamptz) */
/** Database column asof SqlType(timestamp) */
val asof: Rep[java.sql.Timestamp] = column[java.sql.Timestamp]("asof")
}

Expand Down Expand Up @@ -1335,55 +1328,53 @@ trait Tables {

/** Maps whole row to an option. Useful for outer joins. */
def ? =
(branch :: numberOfSlots :: cycle :: Rep.Some(operationId) :: Rep.Some(operationGroupHash) :: Rep.Some(kind) :: level :: delegate :: slots :: nonce :: pkh :: secret :: source :: fee :: counter :: gasLimit :: storageLimit :: publicKey :: amount :: destination :: parameters :: managerPubkey :: balance :: proposal :: spendable :: delegatable :: script :: storage :: status :: consumedGas :: storageSize :: paidStorageSizeDiff :: originatedContracts :: Rep
.Some(
blockHash
) :: Rep.Some(blockLevel) :: ballot :: Rep.Some(internal) :: period :: Rep.Some(timestamp) :: HNil).shaped
.<>(
r =>
OperationsRow(
r(0).asInstanceOf[Option[String]],
r(1).asInstanceOf[Option[Int]],
r(2).asInstanceOf[Option[Int]],
r(3).asInstanceOf[Option[Int]].get,
r(4).asInstanceOf[Option[String]].get,
r(5).asInstanceOf[Option[String]].get,
r(6).asInstanceOf[Option[Int]],
r(7).asInstanceOf[Option[String]],
r(8).asInstanceOf[Option[String]],
r(9).asInstanceOf[Option[String]],
r(10).asInstanceOf[Option[String]],
r(11).asInstanceOf[Option[String]],
r(12).asInstanceOf[Option[String]],
r(13).asInstanceOf[Option[scala.math.BigDecimal]],
r(14).asInstanceOf[Option[scala.math.BigDecimal]],
r(15).asInstanceOf[Option[scala.math.BigDecimal]],
r(16).asInstanceOf[Option[scala.math.BigDecimal]],
r(17).asInstanceOf[Option[String]],
r(18).asInstanceOf[Option[scala.math.BigDecimal]],
r(19).asInstanceOf[Option[String]],
r(20).asInstanceOf[Option[String]],
r(21).asInstanceOf[Option[String]],
r(22).asInstanceOf[Option[scala.math.BigDecimal]],
r(23).asInstanceOf[Option[String]],
r(24).asInstanceOf[Option[Boolean]],
r(25).asInstanceOf[Option[Boolean]],
r(26).asInstanceOf[Option[String]],
r(27).asInstanceOf[Option[String]],
r(28).asInstanceOf[Option[String]],
r(29).asInstanceOf[Option[scala.math.BigDecimal]],
r(30).asInstanceOf[Option[scala.math.BigDecimal]],
r(31).asInstanceOf[Option[scala.math.BigDecimal]],
r(32).asInstanceOf[Option[String]],
r(33).asInstanceOf[Option[String]].get,
r(34).asInstanceOf[Option[Int]].get,
r(35).asInstanceOf[Option[String]],
r(36).asInstanceOf[Option[Boolean]].get,
r(37).asInstanceOf[Option[Int]],
r(38).asInstanceOf[Option[java.sql.Timestamp]].get
),
(_: Any) => throw new Exception("Inserting into ? projection not supported.")
)
(branch :: numberOfSlots :: cycle :: Rep.Some(operationId) :: Rep.Some(operationGroupHash) :: Rep.Some(kind) :: level :: delegate :: slots :: nonce :: pkh :: secret :: source :: fee :: counter :: gasLimit :: storageLimit :: publicKey :: amount :: destination :: parameters :: managerPubkey :: balance :: proposal :: spendable :: delegatable :: script :: storage :: status :: consumedGas :: storageSize :: paidStorageSizeDiff :: originatedContracts :: Rep.Some(
blockHash
) :: Rep.Some(blockLevel) :: ballot :: Rep.Some(internal) :: period :: Rep.Some(timestamp) :: HNil).shaped.<>(
r =>
OperationsRow(
r(0).asInstanceOf[Option[String]],
r(1).asInstanceOf[Option[Int]],
r(2).asInstanceOf[Option[Int]],
r(3).asInstanceOf[Option[Int]].get,
r(4).asInstanceOf[Option[String]].get,
r(5).asInstanceOf[Option[String]].get,
r(6).asInstanceOf[Option[Int]],
r(7).asInstanceOf[Option[String]],
r(8).asInstanceOf[Option[String]],
r(9).asInstanceOf[Option[String]],
r(10).asInstanceOf[Option[String]],
r(11).asInstanceOf[Option[String]],
r(12).asInstanceOf[Option[String]],
r(13).asInstanceOf[Option[scala.math.BigDecimal]],
r(14).asInstanceOf[Option[scala.math.BigDecimal]],
r(15).asInstanceOf[Option[scala.math.BigDecimal]],
r(16).asInstanceOf[Option[scala.math.BigDecimal]],
r(17).asInstanceOf[Option[String]],
r(18).asInstanceOf[Option[scala.math.BigDecimal]],
r(19).asInstanceOf[Option[String]],
r(20).asInstanceOf[Option[String]],
r(21).asInstanceOf[Option[String]],
r(22).asInstanceOf[Option[scala.math.BigDecimal]],
r(23).asInstanceOf[Option[String]],
r(24).asInstanceOf[Option[Boolean]],
r(25).asInstanceOf[Option[Boolean]],
r(26).asInstanceOf[Option[String]],
r(27).asInstanceOf[Option[String]],
r(28).asInstanceOf[Option[String]],
r(29).asInstanceOf[Option[scala.math.BigDecimal]],
r(30).asInstanceOf[Option[scala.math.BigDecimal]],
r(31).asInstanceOf[Option[scala.math.BigDecimal]],
r(32).asInstanceOf[Option[String]],
r(33).asInstanceOf[Option[String]].get,
r(34).asInstanceOf[Option[Int]].get,
r(35).asInstanceOf[Option[String]],
r(36).asInstanceOf[Option[Boolean]].get,
r(37).asInstanceOf[Option[Int]],
r(38).asInstanceOf[Option[java.sql.Timestamp]].get
),
(_: Any) => throw new Exception("Inserting into ? projection not supported.")
)

/** Database column branch SqlType(varchar), Default(None) */
val branch: Rep[Option[String]] = column[Option[String]]("branch", O.Default(None))
Expand Down
6 changes: 3 additions & 3 deletions src/main/scala/tech/cryptonomic/conseil/tezos/TezosDatabaseOperations.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -371,9 +371,9 @@ object TezosDatabaseOperations extends LazyLogging {
}

/** Writes bakers to the database */
def writeVotingRolls(bakers: List[Voting.BakerRolls], block: Block): DBIO[Option[Int]] = {
logger.info(s"""Writing ${bakers.length} bakers to the DB...""")
Tables.Rolls ++= (block, bakers).convertToA[List, Tables.RollsRow]
def writeVotingRolls(bakers: List[(Block, List[Voting.BakerRolls])]): DBIO[Option[Int]] = {
logger.info(s"""Writing ${bakers.size} bakers to the DB...""")
Tables.Rolls ++= bakers.flatMap(_.convertToA[List, Tables.RollsRow])
}

/**
Expand Down
2 changes: 1 addition & 1 deletion src/test/scala/tech/cryptonomic/conseil/tezos/TezosDatabaseOperationsTest.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1042,7 +1042,7 @@ class TezosDatabaseOperationsTest
//write
val writeAndGetRows = for {
_ <- Tables.Blocks += block.convertTo[Tables.BlocksRow]
written <- sut.writeVotingRolls(rolls, block)
written <- sut.writeVotingRolls(List(block -> rolls))
rows <- Tables.Rolls.result
} yield (written, rows)

Expand Down

0 comments on commit eb6df16

Please sign in to comment.