Various improvements (#256)

* Updated proposal Monitor insert
* trying to make assetHolders() faster
* Fixed Smart Contract apis to actually return data
* Added requested NFT apis
* Updated existing NFT apis with additional data
* Started using Kotlin coroutines
* Started integrating with Kotlin protobuf supported functions
* Added Missed Blocks apis for devops
* Fixed tx_gas_fee_volume_* aggregations

closes: #253
closes: #254
closes: #255
closes: #257
closes: #258

CHANGELOG.md

@@ -40,6 +40,9 @@ Ref: https://keepachangelog.com/en/1.0.0/
 * Added ingestion and API for Smart Contract msg types #70
   * API under `/api/v2/smart_contract`
   * Proposal monitoring to find when a `StoreCodeProposal` is created
+* Generating Kotlin protobuf classes using official Protobuf Kotlin DSL #254
+* Added `/api/v2/validators/missed_blocks` and `/api/v2/validators/missed_blocks/distinct` APIs #257 
+  * Used for devops validator monitoring
 
 ### Improvements
 * Added a block tx retry table to store heights that failed to fetch txs #232
@@ -50,6 +53,9 @@ Ref: https://keepachangelog.com/en/1.0.0/
 * Reworked how msg type was being derived #248
   * Now deriving `module`, `type` from `proto.typeUrl` instead of scraping events for possible values
 * Reworked account insert/update to use upsert instead
+* Adding initial use cases for Kotlin Coroutines #255
+* Improved NFT API with additional data #253
+  * Added APIs for specification json responses
 
 ### Bug Fixes
 * Added additional MsgConverter lines to handle older tx msg types
@@ -58,6 +64,7 @@ Ref: https://keepachangelog.com/en/1.0.0/
 * Updated signature finding based on msg type #249
 * Added API response handler for IllegalArgumentExceptions #245
 * Fixed NullPointerException when `session_id_components` was actually null
+* Fixed bug in `update_gas_fee_volume()` procedure #258
 
 ### Data
 * Migration 1.32 - Added `block_tx_retry` table #232
@@ -67,6 +74,8 @@ Ref: https://keepachangelog.com/en/1.0.0/
 * Migration 1.34 - Added unique key to `tx_message_type` #248
   * Upserted records to make records uniform
 * Migration 1.35 - Added `proposal_monitor` table #70
+* Migration 1.36 - Updated `update_gas_fee_volume()` procedure #258
+* Migration 1.37 - Added `missed_block_periods()` function to find missed blocks for inputs #257 
 
 ## [v2.4.0](https://github.com/provenance-io/explorer-service/releases/tag/v2.4.0) - 2021-09-15
 ### Release Name: Bjarni Herjulfsson

buildSrc/src/main/kotlin/Dependencies.kt

@@ -16,6 +16,7 @@ object PluginIds { // please keep this sorted in sections
     const val SpringBoot = "org.springframework.boot"
     const val Protobuf = "com.google.protobuf"
     const val Grpc = "grpc"
+    const val GrpcKt = "grpckt"
 
     // Provenance
     const val ProvenanceDownloadProtos = "io.provenance.download-protos"
@@ -40,7 +41,7 @@ object PluginVersions { // please keep this sorted in sections
 object Versions {
     // kotlin
     const val Kotlin = PluginVersions.Kotlin
-    const val KotlinXCoroutines = "1.5.1"
+    const val KotlinXCoroutines = "1.5.2"
 
     // 3rd Party
     const val ApacheCommonsText = "1.9"
@@ -54,10 +55,10 @@ object Versions {
     const val Logback = "0.1.5"
     const val SpringBoot = PluginVersions.SpringBoot
     const val Swagger = "3.0.0"
-    const val Protobuf = "3.15.0"
+    const val Protobuf = "3.19.1"
     const val Grpc = "1.40.1"
+    const val KotlinGrpc = "1.2.0"
     const val GrpcStarter = "4.5.6"
-    const val ProtocArtifact = "3.17.3"
     const val Postgres = "42.2.23"
 
     // Testing
@@ -66,8 +67,8 @@ object Versions {
     const val Kotest = "4.4.3"
 
     // external protos
-    const val Provenance = "v1.7.0"
-    const val Cosmos = "v0.44.0"
+    const val Provenance = "v1.7.5"
+    const val Cosmos = "v0.44.3"
     const val Wasmd = "v0.19.0"
     const val Ibc = "v1.1.0"
 }
@@ -77,6 +78,7 @@ object Libraries {
     const val KotlinReflect = "org.jetbrains.kotlin:kotlin-reflect:${Versions.Kotlin}"
     const val KotlinStdlib = "org.jetbrains.kotlin:kotlin-stdlib:${Versions.Kotlin}"
     const val KotlinXCoRoutinesCore = "org.jetbrains.kotlinx:kotlinx-coroutines-core:${Versions.KotlinXCoroutines}"
+    const val KotlinXCoRoutinesGuava = "org.jetbrains.kotlinx:kotlinx-coroutines-guava:${Versions.KotlinXCoroutines}"
 
     // 3rd Party
     const val Exposed = "org.jetbrains.exposed:exposed-core:${Versions.Exposed}"
@@ -99,13 +101,15 @@ object Libraries {
     const val LogbackJackson = "ch.qos.logback.contrib:logback-jackson:${Versions.Logback}"
 
     // Protobuf
-    const val ProtobufJava = "com.google.protobuf:protobuf-java:${Versions.Protobuf}"
+    const val ProtobufJavaUtil = "com.google.protobuf:protobuf-java-util:${Versions.Protobuf}"
+    const val ProtobufKotlin = "com.google.protobuf:protobuf-kotlin:${Versions.Protobuf}"
     const val GrpcProtobuf = "io.grpc:grpc-protobuf:${Versions.Grpc}"
     const val GrpcStub = "io.grpc:grpc-stub:${Versions.Grpc}"
-    const val ProtocArtifact = "com.google.protobuf:protoc:${Versions.ProtocArtifact}"
+    const val GrpcKotlinStub = "io.grpc:grpc-kotlin-stub:${Versions.KotlinGrpc}"
+    const val ProtocArtifact = "com.google.protobuf:protoc:${Versions.Protobuf}"
     const val GrpcArtifact = "io.grpc:protoc-gen-grpc-java:${Versions.Grpc}"
+    const val GrpcKotlinArtifact = "io.grpc:protoc-gen-grpc-kotlin:${Versions.KotlinGrpc}:jdk7@jar"
     const val GrpcNetty = "io.grpc:grpc-netty:${Versions.Grpc}"
-    const val GrpcStart = "io.github.lognet:grpc-spring-boot-starter:${Versions.GrpcStarter}"
 
     // Spring
     const val SpringBootDevTools = "org.springframework.boot:spring-boot-devtools:${Versions.SpringBoot}"

buildSrc/src/main/kotlin/io/provenance/DownloadProtosTask.kt

@@ -28,14 +28,14 @@ open class DownloadProtosTask : DefaultTask() {
 
     @Option(
         option = "provenance-version",
-        description = "Provenance release version (e.g. v1.7.0)"
+        description = "Provenance release version (e.g. v1.7.5)"
     )
     @Input
     var provenanceVersion: String? = null
 
     @Option(
         option = "cosmos-version",
-        description = "Cosmos release version (e.g. v0.44.0)"
+        description = "Cosmos release version (e.g. v0.44.3)"
     )
     @Input
     var cosmosVersion: String? = null

database/src/main/resources/db/migration/V1_36__Update_gas_fee_volumes.sql

@@ -0,0 +1,65 @@
+SELECT 'Update update_gas_fee_volume() procedure' AS comment;
+-- Update gas stats with latest data
+CREATE OR REPLACE PROCEDURE update_gas_fee_volume()
+    LANGUAGE plpgsql
+AS
+$$
+DECLARE
+    unprocessed_daily_times  TIMESTAMP[];
+    unprocessed_hourly_times TIMESTAMP[];
+    unprocessed_ids          INT[];
+BEGIN
+    -- Collect unprocessed message timestamps (grouped by day and hour)
+    SELECT array_agg(DISTINCT date_trunc('DAY', tx_timestamp)) FROM tx_gas_cache WHERE processed = false INTO unprocessed_daily_times;
+    SELECT array_agg(DISTINCT date_trunc('HOUR', tx_timestamp)) FROM tx_gas_cache WHERE processed = false INTO unprocessed_hourly_times;
+    SELECT array_agg(id) FROM tx_gas_cache WHERE processed = false INTO unprocessed_ids;
+
+    -- Update daily stats for unprocessed messages
+    -- Reprocess daily stats for the day that unprocessed messages fall in
+    INSERT
+    INTO tx_gas_fee_volume_day
+    SELECT date_trunc('DAY', tx_timestamp)                                                              AS tx_timestamp,
+           sum(gas_wanted)                                                                              AS gas_wanted,
+           sum(gas_used)                                                                                AS gas_used,
+           sum((tx_v2 -> 'tx' -> 'auth_info' -> 'fee' -> 'amount' -> 0 ->> 'amount')::DOUBLE PRECISION) AS fee_amount
+    FROM tx_cache
+    WHERE date_trunc('DAY', tx_timestamp) = ANY (unprocessed_daily_times)
+    GROUP BY date_trunc('DAY', tx_timestamp)
+    ON CONFLICT (tx_timestamp)
+        DO UPDATE
+        SET gas_wanted = excluded.gas_wanted,
+            gas_used = excluded.gas_used,
+            fee_amount = excluded.fee_amount;
+
+    -- Update hourly stats for unprocessed messages
+    -- Reprocess hourly stats for the day that unprocessed messages fall in
+    INSERT
+    INTO tx_gas_fee_volume_hour
+    SELECT date_trunc('HOUR', tx_timestamp)                                                             AS tx_timestamp,
+           sum(gas_wanted)                                                                              AS gas_wanted,
+           sum(gas_used)                                                                                AS gas_used,
+           sum((tx_v2 -> 'tx' -> 'auth_info' -> 'fee' -> 'amount' -> 0 ->> 'amount')::DOUBLE PRECISION) AS fee_amount
+    FROM tx_cache
+    WHERE date_trunc('HOUR', tx_timestamp) = ANY (unprocessed_hourly_times)
+    GROUP BY date_trunc('HOUR', tx_timestamp)
+    ON CONFLICT (tx_timestamp)
+        DO UPDATE
+        SET gas_wanted = excluded.gas_wanted,
+            gas_used = excluded.gas_used,
+            fee_amount = excluded.fee_amount;
+
+    -- Update complete. Now mark records as 'processed'.
+    UPDATE tx_gas_cache
+    SET processed = true
+    WHERE id = ANY (unprocessed_ids);
+
+    RAISE INFO 'UPDATED gas fee volume';
+END;
+$$;
+
+SELECT 'Updating tx_gas_cache to reset' AS comment;
+UPDATE tx_gas_cache
+SET processed = false;
+
+SELECT 'Calling procedure to bring up to date' AS comment;
+CALL update_gas_fee_volume();

database/src/main/resources/db/migration/V1_37__Add_missed_blocks_period_function.sql

@@ -0,0 +1,35 @@
+CREATE OR REPLACE FUNCTION missed_block_periods(fromHeight int, toHeight int, address varchar(128) = NULL)
+    RETURNS TABLE (val_cons_address varchar(128), blocks int[]) AS
+$func$
+DECLARE
+    r  missed_blocks; -- use table type as row variable
+    r0 missed_blocks;
+BEGIN
+
+    FOR r IN
+        SELECT
+            *
+        FROM missed_blocks t
+        WHERE t.block_height between fromHeight and toHeight
+          AND (address IS NULL OR t.val_cons_address = address)
+        ORDER BY t.val_cons_address, t.block_height
+        LOOP
+            IF ( r.val_cons_address,  r.block_height)
+                <> (r0.val_cons_address, r0.block_height + 1) THEN -- not true for first row
+
+                RETURN QUERY
+                    SELECT r0.val_cons_address, blocks; -- output row
+
+                blocks := ARRAY[r.block_height];     -- start new array
+            ELSE
+                blocks := blocks || r.block_height;   -- add to array - year can be NULL, too
+            END IF;
+
+            r0 := r;                       -- remember last row
+        END LOOP;
+
+    RETURN QUERY                      -- output last iteration
+        SELECT r0.val_cons_address, blocks;
+
+END
+$func$ LANGUAGE plpgsql;

docker/docker-compose.yml

@@ -23,7 +23,7 @@ services:
       - DB_PORT=5432
       - DB_NAME=explorer
       - DB_SCHEMA=explorer
-      - DB_CONNECTION_POOL_SIZE=5
+      - DB_CONNECTION_POOL_SIZE=40
       - SPOTLIGHT_TTL_MS=5000
       - INITIAL_HIST_DAY_COUNT=14
       - EXPLORER_MAINNET=false

docs/nft/NFTs.md

@@ -117,6 +117,8 @@ Columns: ✅
 * name -> from scope spec
 * scope spec addr
 * Last updated from tx
+* isOwner -> boolean to contain the given address
+* isValueOwner -> boolean to match the given address
 
 ## Asset - NFT count
 * fetch count of NFTs owned by asset holding address  ✅

docs/nft/OwnerUpdates.md

@@ -0,0 +1 @@
+Track when a scope owner/value owner/data access is initiated or updated.

gradle.properties

@@ -1,3 +1,4 @@
 kotlin.code.style=official
 kapt.use.worker.api=false
 kapt.incremental.apt=false
+org.gradle.jvmargs=-Xmx4096M

proto/README.md

@@ -30,16 +30,16 @@ in the `./buildSrc/src/main/kotlin/Dependencies.kt` file:
 
 ```kotlin
     //external protos
-    const val Provenance = "v1.7.0"
-    const val Cosmos = "v0.44.0"
+    const val Provenance = "v1.7.5"
+    const val Cosmos = "v0.44.3"
     const val Wasmd = "v0.19.0"
     const val Ibc = "v1.1.0"
 ```
 
 To manually specify the versions run this `gradle` task  *from the root project directory*:
 
 ```bash
-./gradlew downloadProtos --provenance-version v1.7.0 --cosmos-version v0.44.0 --wasmd-version v0.19.0 --ibc-version v1.1.0
+./gradlew downloadProtos --provenance-version v1.7.5 --cosmos-version v0.44.3 --wasmd-version v0.19.0 --ibc-version v1.1.0
 ```
 
 > The proto download process does not need to be run very often, 
@@ -51,5 +51,5 @@ To manually specify the versions run this `gradle` task  *from the root project
 Once the protos have been downloaded, run the `gradle` task *from the root project directory*:
 
 ```bash
-./gradlew clean proto:build
+./gradlew clean proto:generateProto
 ```

proto/build.gradle.kts

@@ -7,13 +7,15 @@ import com.google.protobuf.gradle.protoc
 
 sourceSets.main {
     proto.srcDirs("../third_party/proto/")
-    java.srcDirs("build/generated/source/proto/main/java")
 }
 
 dependencies {
-    api(Libraries.ProtobufJava)
-    api(Libraries.GrpcStub)
+//    protobuf(files("cosmWasm-v0.17.0.tar.gz"))
+    api(Libraries.ProtobufJavaUtil)
+    implementation(Libraries.ProtobufKotlin)
+    api(Libraries.GrpcKotlinStub)
     api(Libraries.GrpcProtobuf)
+    implementation(Libraries.GrpcStub)
 
     if (JavaVersion.current().isJava9Compatible) {
         // Workaround for @javax.annotation.Generated
@@ -22,6 +24,12 @@ dependencies {
     }
 }
 
+tasks.withType<org.jetbrains.kotlin.gradle.tasks.KotlinCompile>().all {
+    kotlinOptions {
+        freeCompilerArgs = listOf("-Xopt-in=kotlin.RequiresOptIn")
+    }
+}
+
 protobuf {
     protoc {
         // The artifact spec for the Protobuf Compiler
@@ -34,12 +42,18 @@ protobuf {
         id(PluginIds.Grpc) {
             artifact = Libraries.GrpcArtifact
         }
+        id(PluginIds.GrpcKt) {
+            artifact = Libraries.GrpcKotlinArtifact
+        }
     }
     generateProtoTasks {
-        ofSourceSet("main").forEach {
+        all().forEach {
             it.plugins {
-                // Apply the "grpc" plugin whose spec is defined above, without options.
                 id(PluginIds.Grpc)
+                id(PluginIds.GrpcKt)
+            }
+            it.builtins {
+                id(PluginIds.Kotlin)
             }
         }
     }
@@ -51,3 +65,13 @@ tasks.register<io.provenance.DownloadProtosTask>("downloadProtos") {
     wasmdVersion = Versions.Wasmd
     ibcVersion = Versions.Ibc
 }
+
+//tasks.register("downloadTest"){
+//    mapOf("cosmWasm-v0.17.0.tar.gz" to "https://github.com/CosmWasm/wasmd/tarball/v0.17.0")
+//        .forEach { (k, v) -> download(v,k) }
+//}
+//
+//fun download(url : String, path : String){
+//    val destFile = File(path)
+//    ant.invokeMethod("get", mapOf("src" to url, "dest" to destFile))
+//}

service/build.gradle.kts

@@ -17,15 +17,6 @@ sourceSets {
             resources.srcDirs("src/test/resources")
         }
     }
-//    create("integrationTest") {
-//        withConvention(KotlinSourceSet::class) {
-//            kotlin.srcDirs("src/test/kotlin")
-//            compileClasspath += sourceSets.main.get().output + sourceSets.test.get().output +
-//                configurations.testRuntimeOnly + configurations.testImplementation
-//            runtimeClasspath += output + compileClasspath + test.get().output
-//            resources.srcDirs(file("src/test/resources"))
-//        }
-//    }
 }
 
 configurations.all {
@@ -45,12 +36,12 @@ dependencies {
 
     api(Libraries.BouncyCastle)
     api(Libraries.KotlinXCoRoutinesCore)
+    api(Libraries.KotlinXCoRoutinesGuava)
     api(Libraries.ApacheCommonsText)
     api(Libraries.Khttp)
     implementation(Libraries.KaseChange)
 
     implementation(Libraries.GrpcNetty)
-    implementation(Libraries.GrpcStart)
 
     api(Libraries.LogbackCore)
     api(Libraries.LogbackClassic)

service/src/main/kotlin/io/provenance/explorer/config/GrpcClientConfig.kt

@@ -1,5 +1,6 @@
 package io.provenance.explorer.config
 
+import kotlinx.coroutines.sync.Semaphore
 import org.springframework.boot.context.properties.EnableConfigurationProperties
 import org.springframework.context.annotation.Bean
 import org.springframework.context.annotation.Configuration
@@ -13,4 +14,7 @@ class GrpcClientConfig(val props: ExplorerProperties) {
 
     @Bean
     fun channelUri() = URI(props.pbUrl)
+
+    @Bean
+    fun semaphore() = Semaphore(System.getenv("GRPC_CONCURRENCY")?.toInt() ?: 20)
 }