Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

953 - Adding possibility of using "countDistinct" in aggregation #1027

Merged
merged 2 commits into from
Sep 14, 2021
Merged

953 - Adding possibility of using "countDistinct" in aggregation #1027

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
4148c38
Adding possibility of using "distinct" in the aggregation
piotrkosecki Aug 9, 2021
2b50a38
updates to aggregation after consultations with Mike
piotrkosecki Aug 12, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
59 changes: 46 additions & 13 deletions ...ala/tech/cryptonomic/conseil/api/routes/platform/data/tezos/TezosDataOperationsTest.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,18 +6,11 @@ import com.softwaremill.diffx.scalatest.DiffMatcher
import org.scalatest.concurrent.IntegrationPatience
import slick.jdbc.PostgresProfile.api._
import tech.cryptonomic.conseil.api.TezosInMemoryDatabaseSetup
import tech.cryptonomic.conseil.common.generic.chain.DataTypes.{Query, _}
import tech.cryptonomic.conseil.common.generic.chain.DataTypes.{Query, SimpleField, _}
import tech.cryptonomic.conseil.common.testkit.{ConseilSpec, InMemoryDatabase}
import tech.cryptonomic.conseil.common.tezos.Tables.{
AccountsHistoryRow,
AccountsRow,
BlocksRow,
FeesRow,
OperationGroupsRow,
OperationsRow
}
import tech.cryptonomic.conseil.common.tezos.Tables.{AccountsHistoryRow, AccountsRow, BlocksRow, FeesRow, OperationGroupsRow, OperationsRow}
import tech.cryptonomic.conseil.common.tezos.{Fork, Tables}
import tech.cryptonomic.conseil.common.tezos.TezosTypes.{makeAccountId, TezosBlockHash}
import tech.cryptonomic.conseil.common.tezos.TezosTypes.{TezosBlockHash, makeAccountId}

import scala.concurrent.duration._

Expand Down Expand Up @@ -2111,7 +2104,7 @@ class TezosDataOperationsTest
Aggregation(
field = "medium",
function = AggregationType.count,
Some(
predicate = Some(
AggregationPredicate(
operation = OperationType.gt,
set = List(1),
Expand Down Expand Up @@ -2169,7 +2162,7 @@ class TezosDataOperationsTest
Aggregation(
field = "medium",
function = AggregationType.count,
Some(
predicate = Some(
AggregationPredicate(
operation = OperationType.gt,
set = List(0),
Expand All @@ -2180,7 +2173,7 @@ class TezosDataOperationsTest
Aggregation(
field = "low",
function = AggregationType.sum,
Some(
predicate = Some(
AggregationPredicate(
operation = OperationType.eq,
set = List(0),
Expand Down Expand Up @@ -2271,6 +2264,46 @@ class TezosDataOperationsTest

}

"aggregate with distinct count aggregation" in {
val feesTmp = List(
FeesRow(1, 1, 1, Timestamp.valueOf("2000-01-01 00:00:00"), "kind", forkId = Fork.mainForkId),
FeesRow(2, 1, 1, Timestamp.valueOf("2000-01-02 00:00:00"), "kind", forkId = Fork.mainForkId),
FeesRow(3, 2, 1, Timestamp.valueOf("2000-01-02 00:00:00"), "kind", forkId = Fork.mainForkId),
FeesRow(4, 2, 1, Timestamp.valueOf("2000-01-02 00:00:00"), "kind", forkId = Fork.mainForkId),
FeesRow(5, 3, 2, Timestamp.valueOf("2000-01-03 00:00:00"), "kind", forkId = Fork.mainForkId),
FeesRow(6, 3, 2, Timestamp.valueOf("2000-01-03 00:00:00"), "kind", forkId = Fork.mainForkId)
)

val aggregate = List(
Aggregation("low", AggregationType.count),
Aggregation("medium", AggregationType.countDistinct)
)

val populateAndTest = for {
_ <- Tables.Fees ++= feesTmp
found <- sut.selectWithPredicates(
"tezos",
table = Tables.Fees.baseTableRow.tableName,
columns = List(SimpleField("low"), SimpleField("medium"), SimpleField("high")),
predicates = List.empty,
ordering = List(QueryOrdering("count_distinct_medium", OrderDirection.desc)),
aggregation = aggregate,
temporalPartition = None,
snapshot = None,
outputType = OutputType.json,
limit = 10
)
} yield found

val result = dbHandler.run(populateAndTest.transactionally).futureValue

result shouldBe List(
Map("count_low" -> Some(4), "count_distinct_medium" -> Some(2), "high" -> Some(1)),
Map("count_low" -> Some(2), "count_distinct_medium" -> Some(1), "high" -> Some(2))
)
}


"map date with datePart aggregation when it is only type of aggregation" in {
val feesTmp = List(
FeesRow(0, 1, 4, Timestamp.valueOf("2000-01-01 00:00:00"), "kind", forkId = Fork.mainForkId),
Expand Down
4 changes: 2 additions & 2 deletions conseil-common/src/main/scala/tech/cryptonomic/conseil/common/generic/chain/DataTypes.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ object DataTypes {
/** Method checks if type can be aggregated */
lazy val canBeAggregated: DataType => AggregationType => Boolean = { dataType =>
{
case AggregationType.count => true
case AggregationType.count | AggregationType.countDistinct => true
case AggregationType.max | AggregationType.min =>
Set(DataType.Decimal, DataType.Int, DataType.LargeInt, DataType.DateTime, DataType.Currency)(dataType)
case AggregationType.avg | AggregationType.sum =>
Expand Down Expand Up @@ -249,7 +249,7 @@ object DataTypes {
/** Helper method for extracting prefixes needed for SQL */
def prefixes: List[String] = values.toList.map(_.toString + "_")
type AggregationType = Value
val sum, count, max, min, avg = Value
val sum, count, max, min, avg, countDistinct = Value
}

/** Enumeration of aggregation functions */
Expand Down
2 changes: 2 additions & 0 deletions conseil-common/src/main/scala/tech/cryptonomic/conseil/common/util/DatabaseUtil.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -326,6 +326,7 @@ object DatabaseUtil {
aggregationType match {
case AggregationType.sum => s"SUM($column)"
case AggregationType.count => s"COUNT($column)"
case AggregationType.countDistinct => s"COUNT(DISTINCT $column)"
case AggregationType.max => s"MAX($column)"
case AggregationType.min => s"MIN($column)"
case AggregationType.avg => s"AVG($column)"
Expand All @@ -336,6 +337,7 @@ object DatabaseUtil {
aggregationType match {
case AggregationType.sum => s"sum_$column"
case AggregationType.count => s"count_$column"
case AggregationType.countDistinct => s"count_distinct_$column"
case AggregationType.max => s"max_$column"
case AggregationType.min => s"min_$column"
case AggregationType.avg => s"avg_$column"
Expand Down
6 changes: 5 additions & 1 deletion ...rre/src/main/scala/tech/cryptonomic/conseil/indexer/tezos/bigmaps/BigMapsOperations.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ import tech.cryptonomic.conseil.common.tezos.TezosTypes.BlockTagged.fromBlockDat
*/
case class BigMapsOperations[Profile <: ExPostgresProfile](profile: Profile) extends ConseilLogSupport {
import profile.api._
import io.scalaland.chimney.dsl._

/** Create an action to find and copy big maps based on the diff contained in the blocks
*
Expand Down Expand Up @@ -95,7 +96,10 @@ case class BigMapsOperations[Profile <: ExPostgresProfile](profile: Profile) ext
DBIO.sequence {
List(
Tables.BigMapContents.insertOrUpdateAll(rowsToWrite),
Tables.BigMapContentsHistory ++= updateData.map(Tables.BigMapContentsHistoryRow.tupled).distinct
Tables.BigMapContentsHistory ++= updateData
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Note to self: This fixes a compilation issue.

.map(BigMapContentsRow.tupled)
.map(_.transformInto[Tables.BigMapContentsHistoryRow])
.distinct
)
}
}
Expand Down