From ff3f14312be1dea099e0de8153765dc80ddb6fce Mon Sep 17 00:00:00 2001 From: Carolyn Russell Date: Thu, 18 Nov 2021 16:01:08 -0700 Subject: [PATCH] Various improvements (#256) * Updated proposal Monitor insert * trying to make assetHolders() faster * Fixed Smart Contract apis to actually return data * Added requested NFT apis * Updated existing NFT apis with additional data * Started using Kotlin coroutines * Started integrating with Kotlin protobuf supported functions * Added Missed Blocks apis for devops * Fixed tx_gas_fee_volume_* aggregations closes: #253 closes: #254 closes: #255 closes: #257 closes: #258 --- CHANGELOG.md | 9 ++ buildSrc/src/main/kotlin/Dependencies.kt | 20 +-- .../io/provenance/DownloadProtosTask.kt | 4 +- .../V1_36__Update_gas_fee_volumes.sql | 65 +++++++++ ..._37__Add_missed_blocks_period_function.sql | 35 +++++ docker/docker-compose.yml | 2 +- docs/nft/NFTs.md | 2 + docs/nft/OwnerUpdates.md | 1 + gradle.properties | 1 + proto/README.md | 8 +- proto/build.gradle.kts | 34 ++++- service/build.gradle.kts | 11 +- .../explorer/config/GrpcClientConfig.kt | 4 + .../explorer/config/GrpcLoggingInterceptor.kt | 4 +- .../explorer/domain/core/MetadataAddress.kt | 5 + .../explorer/domain/entities/Blocks.kt | 31 +++- .../explorer/domain/entities/Governance.kt | 4 +- .../explorer/domain/entities/Nfts.kt | 61 ++++++++ .../domain/entities/SmartContracts.kt | 8 ++ .../explorer/domain/entities/Validators.kt | 13 +- .../MessageTranslationExtensions.kt | 46 ++++-- .../domain/models/explorer/AssetModels.kt | 3 +- .../domain/models/explorer/CommonModels.kt | 3 + .../domain/models/explorer/NftModels.kt | 26 +++- .../models/explorer/SmartContractModels.kt | 17 +++ .../domain/models/explorer/ValidatorModels.kt | 28 ++++ .../explorer/grpc/extensions/Domain.kt | 17 +++ .../explorer/grpc/v1/AccountGrpcClient.kt | 133 +++++++++--------- .../explorer/grpc/v1/AttributeGrpcClient.kt | 53 ++++--- .../explorer/grpc/v1/MarkerGrpcClient.kt | 66 +++++---- .../explorer/grpc/v1/MetadataGrpcClient.kt | 91 ++++++------ .../explorer/grpc/v1/TransactionGrpcClient.kt | 4 +- .../explorer/service/AccountService.kt | 87 +++++++----- .../explorer/service/AssetService.kt | 69 ++++----- .../explorer/service/ExplorerService.kt | 11 +- .../provenance/explorer/service/NftService.kt | 75 +++++++--- .../explorer/service/SmartContractService.kt | 40 ++++-- .../explorer/service/TransactionService.kt | 6 +- .../explorer/service/ValidatorService.kt | 67 ++++++++- .../explorer/service/async/AsyncCaching.kt | 2 +- .../service/utility/UtilityService.kt | 37 ++--- .../explorer/web/v2/MigrationController.kt | 2 +- .../explorer/web/v2/NftController.kt | 12 ++ .../web/v2/SmartContractController.kt | 9 +- .../explorer/web/v2/UtilityController.kt | 2 +- .../explorer/web/v2/ValidatorController.kt | 14 ++ 46 files changed, 901 insertions(+), 341 deletions(-) create mode 100644 database/src/main/resources/db/migration/V1_36__Update_gas_fee_volumes.sql create mode 100644 database/src/main/resources/db/migration/V1_37__Add_missed_blocks_period_function.sql create mode 100644 docs/nft/OwnerUpdates.md create mode 100644 service/src/main/kotlin/io/provenance/explorer/domain/models/explorer/SmartContractModels.kt diff --git a/CHANGELOG.md b/CHANGELOG.md index ac9ea088..880e0b5b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,6 +40,9 @@ Ref: https://keepachangelog.com/en/1.0.0/ * Added ingestion and API for Smart Contract msg types #70 * API under `/api/v2/smart_contract` * Proposal monitoring to find when a `StoreCodeProposal` is created +* Generating Kotlin protobuf classes using official Protobuf Kotlin DSL #254 +* Added `/api/v2/validators/missed_blocks` and `/api/v2/validators/missed_blocks/distinct` APIs #257 + * Used for devops validator monitoring ### Improvements * Added a block tx retry table to store heights that failed to fetch txs #232 @@ -50,6 +53,9 @@ Ref: https://keepachangelog.com/en/1.0.0/ * Reworked how msg type was being derived #248 * Now deriving `module`, `type` from `proto.typeUrl` instead of scraping events for possible values * Reworked account insert/update to use upsert instead +* Adding initial use cases for Kotlin Coroutines #255 +* Improved NFT API with additional data #253 + * Added APIs for specification json responses ### Bug Fixes * Added additional MsgConverter lines to handle older tx msg types @@ -58,6 +64,7 @@ Ref: https://keepachangelog.com/en/1.0.0/ * Updated signature finding based on msg type #249 * Added API response handler for IllegalArgumentExceptions #245 * Fixed NullPointerException when `session_id_components` was actually null +* Fixed bug in `update_gas_fee_volume()` procedure #258 ### Data * Migration 1.32 - Added `block_tx_retry` table #232 @@ -67,6 +74,8 @@ Ref: https://keepachangelog.com/en/1.0.0/ * Migration 1.34 - Added unique key to `tx_message_type` #248 * Upserted records to make records uniform * Migration 1.35 - Added `proposal_monitor` table #70 +* Migration 1.36 - Updated `update_gas_fee_volume()` procedure #258 +* Migration 1.37 - Added `missed_block_periods()` function to find missed blocks for inputs #257 ## [v2.4.0](https://github.com/provenance-io/explorer-service/releases/tag/v2.4.0) - 2021-09-15 ### Release Name: Bjarni Herjulfsson diff --git a/buildSrc/src/main/kotlin/Dependencies.kt b/buildSrc/src/main/kotlin/Dependencies.kt index fa65662a..fac65a3a 100644 --- a/buildSrc/src/main/kotlin/Dependencies.kt +++ b/buildSrc/src/main/kotlin/Dependencies.kt @@ -16,6 +16,7 @@ object PluginIds { // please keep this sorted in sections const val SpringBoot = "org.springframework.boot" const val Protobuf = "com.google.protobuf" const val Grpc = "grpc" + const val GrpcKt = "grpckt" // Provenance const val ProvenanceDownloadProtos = "io.provenance.download-protos" @@ -40,7 +41,7 @@ object PluginVersions { // please keep this sorted in sections object Versions { // kotlin const val Kotlin = PluginVersions.Kotlin - const val KotlinXCoroutines = "1.5.1" + const val KotlinXCoroutines = "1.5.2" // 3rd Party const val ApacheCommonsText = "1.9" @@ -54,10 +55,10 @@ object Versions { const val Logback = "0.1.5" const val SpringBoot = PluginVersions.SpringBoot const val Swagger = "3.0.0" - const val Protobuf = "3.15.0" + const val Protobuf = "3.19.1" const val Grpc = "1.40.1" + const val KotlinGrpc = "1.2.0" const val GrpcStarter = "4.5.6" - const val ProtocArtifact = "3.17.3" const val Postgres = "42.2.23" // Testing @@ -66,8 +67,8 @@ object Versions { const val Kotest = "4.4.3" // external protos - const val Provenance = "v1.7.0" - const val Cosmos = "v0.44.0" + const val Provenance = "v1.7.5" + const val Cosmos = "v0.44.3" const val Wasmd = "v0.19.0" const val Ibc = "v1.1.0" } @@ -77,6 +78,7 @@ object Libraries { const val KotlinReflect = "org.jetbrains.kotlin:kotlin-reflect:${Versions.Kotlin}" const val KotlinStdlib = "org.jetbrains.kotlin:kotlin-stdlib:${Versions.Kotlin}" const val KotlinXCoRoutinesCore = "org.jetbrains.kotlinx:kotlinx-coroutines-core:${Versions.KotlinXCoroutines}" + const val KotlinXCoRoutinesGuava = "org.jetbrains.kotlinx:kotlinx-coroutines-guava:${Versions.KotlinXCoroutines}" // 3rd Party const val Exposed = "org.jetbrains.exposed:exposed-core:${Versions.Exposed}" @@ -99,13 +101,15 @@ object Libraries { const val LogbackJackson = "ch.qos.logback.contrib:logback-jackson:${Versions.Logback}" // Protobuf - const val ProtobufJava = "com.google.protobuf:protobuf-java:${Versions.Protobuf}" + const val ProtobufJavaUtil = "com.google.protobuf:protobuf-java-util:${Versions.Protobuf}" + const val ProtobufKotlin = "com.google.protobuf:protobuf-kotlin:${Versions.Protobuf}" const val GrpcProtobuf = "io.grpc:grpc-protobuf:${Versions.Grpc}" const val GrpcStub = "io.grpc:grpc-stub:${Versions.Grpc}" - const val ProtocArtifact = "com.google.protobuf:protoc:${Versions.ProtocArtifact}" + const val GrpcKotlinStub = "io.grpc:grpc-kotlin-stub:${Versions.KotlinGrpc}" + const val ProtocArtifact = "com.google.protobuf:protoc:${Versions.Protobuf}" const val GrpcArtifact = "io.grpc:protoc-gen-grpc-java:${Versions.Grpc}" + const val GrpcKotlinArtifact = "io.grpc:protoc-gen-grpc-kotlin:${Versions.KotlinGrpc}:jdk7@jar" const val GrpcNetty = "io.grpc:grpc-netty:${Versions.Grpc}" - const val GrpcStart = "io.github.lognet:grpc-spring-boot-starter:${Versions.GrpcStarter}" // Spring const val SpringBootDevTools = "org.springframework.boot:spring-boot-devtools:${Versions.SpringBoot}" diff --git a/buildSrc/src/main/kotlin/io/provenance/DownloadProtosTask.kt b/buildSrc/src/main/kotlin/io/provenance/DownloadProtosTask.kt index 353324d9..cebd2ff7 100644 --- a/buildSrc/src/main/kotlin/io/provenance/DownloadProtosTask.kt +++ b/buildSrc/src/main/kotlin/io/provenance/DownloadProtosTask.kt @@ -28,14 +28,14 @@ open class DownloadProtosTask : DefaultTask() { @Option( option = "provenance-version", - description = "Provenance release version (e.g. v1.7.0)" + description = "Provenance release version (e.g. v1.7.5)" ) @Input var provenanceVersion: String? = null @Option( option = "cosmos-version", - description = "Cosmos release version (e.g. v0.44.0)" + description = "Cosmos release version (e.g. v0.44.3)" ) @Input var cosmosVersion: String? = null diff --git a/database/src/main/resources/db/migration/V1_36__Update_gas_fee_volumes.sql b/database/src/main/resources/db/migration/V1_36__Update_gas_fee_volumes.sql new file mode 100644 index 00000000..984fbb2c --- /dev/null +++ b/database/src/main/resources/db/migration/V1_36__Update_gas_fee_volumes.sql @@ -0,0 +1,65 @@ +SELECT 'Update update_gas_fee_volume() procedure' AS comment; +-- Update gas stats with latest data +CREATE OR REPLACE PROCEDURE update_gas_fee_volume() + LANGUAGE plpgsql +AS +$$ +DECLARE + unprocessed_daily_times TIMESTAMP[]; + unprocessed_hourly_times TIMESTAMP[]; + unprocessed_ids INT[]; +BEGIN + -- Collect unprocessed message timestamps (grouped by day and hour) + SELECT array_agg(DISTINCT date_trunc('DAY', tx_timestamp)) FROM tx_gas_cache WHERE processed = false INTO unprocessed_daily_times; + SELECT array_agg(DISTINCT date_trunc('HOUR', tx_timestamp)) FROM tx_gas_cache WHERE processed = false INTO unprocessed_hourly_times; + SELECT array_agg(id) FROM tx_gas_cache WHERE processed = false INTO unprocessed_ids; + + -- Update daily stats for unprocessed messages + -- Reprocess daily stats for the day that unprocessed messages fall in + INSERT + INTO tx_gas_fee_volume_day + SELECT date_trunc('DAY', tx_timestamp) AS tx_timestamp, + sum(gas_wanted) AS gas_wanted, + sum(gas_used) AS gas_used, + sum((tx_v2 -> 'tx' -> 'auth_info' -> 'fee' -> 'amount' -> 0 ->> 'amount')::DOUBLE PRECISION) AS fee_amount + FROM tx_cache + WHERE date_trunc('DAY', tx_timestamp) = ANY (unprocessed_daily_times) + GROUP BY date_trunc('DAY', tx_timestamp) + ON CONFLICT (tx_timestamp) + DO UPDATE + SET gas_wanted = excluded.gas_wanted, + gas_used = excluded.gas_used, + fee_amount = excluded.fee_amount; + + -- Update hourly stats for unprocessed messages + -- Reprocess hourly stats for the day that unprocessed messages fall in + INSERT + INTO tx_gas_fee_volume_hour + SELECT date_trunc('HOUR', tx_timestamp) AS tx_timestamp, + sum(gas_wanted) AS gas_wanted, + sum(gas_used) AS gas_used, + sum((tx_v2 -> 'tx' -> 'auth_info' -> 'fee' -> 'amount' -> 0 ->> 'amount')::DOUBLE PRECISION) AS fee_amount + FROM tx_cache + WHERE date_trunc('HOUR', tx_timestamp) = ANY (unprocessed_hourly_times) + GROUP BY date_trunc('HOUR', tx_timestamp) + ON CONFLICT (tx_timestamp) + DO UPDATE + SET gas_wanted = excluded.gas_wanted, + gas_used = excluded.gas_used, + fee_amount = excluded.fee_amount; + + -- Update complete. Now mark records as 'processed'. + UPDATE tx_gas_cache + SET processed = true + WHERE id = ANY (unprocessed_ids); + + RAISE INFO 'UPDATED gas fee volume'; +END; +$$; + +SELECT 'Updating tx_gas_cache to reset' AS comment; +UPDATE tx_gas_cache +SET processed = false; + +SELECT 'Calling procedure to bring up to date' AS comment; +CALL update_gas_fee_volume(); diff --git a/database/src/main/resources/db/migration/V1_37__Add_missed_blocks_period_function.sql b/database/src/main/resources/db/migration/V1_37__Add_missed_blocks_period_function.sql new file mode 100644 index 00000000..9e557553 --- /dev/null +++ b/database/src/main/resources/db/migration/V1_37__Add_missed_blocks_period_function.sql @@ -0,0 +1,35 @@ +CREATE OR REPLACE FUNCTION missed_block_periods(fromHeight int, toHeight int, address varchar(128) = NULL) + RETURNS TABLE (val_cons_address varchar(128), blocks int[]) AS +$func$ +DECLARE + r missed_blocks; -- use table type as row variable + r0 missed_blocks; +BEGIN + + FOR r IN + SELECT + * + FROM missed_blocks t + WHERE t.block_height between fromHeight and toHeight + AND (address IS NULL OR t.val_cons_address = address) + ORDER BY t.val_cons_address, t.block_height + LOOP + IF ( r.val_cons_address, r.block_height) + <> (r0.val_cons_address, r0.block_height + 1) THEN -- not true for first row + + RETURN QUERY + SELECT r0.val_cons_address, blocks; -- output row + + blocks := ARRAY[r.block_height]; -- start new array + ELSE + blocks := blocks || r.block_height; -- add to array - year can be NULL, too + END IF; + + r0 := r; -- remember last row + END LOOP; + + RETURN QUERY -- output last iteration + SELECT r0.val_cons_address, blocks; + +END +$func$ LANGUAGE plpgsql; diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 7c0851d0..c25c1824 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -23,7 +23,7 @@ services: - DB_PORT=5432 - DB_NAME=explorer - DB_SCHEMA=explorer - - DB_CONNECTION_POOL_SIZE=5 + - DB_CONNECTION_POOL_SIZE=40 - SPOTLIGHT_TTL_MS=5000 - INITIAL_HIST_DAY_COUNT=14 - EXPLORER_MAINNET=false diff --git a/docs/nft/NFTs.md b/docs/nft/NFTs.md index 5f82b771..0376870f 100644 --- a/docs/nft/NFTs.md +++ b/docs/nft/NFTs.md @@ -117,6 +117,8 @@ Columns: ✅ * name -> from scope spec * scope spec addr * Last updated from tx +* isOwner -> boolean to contain the given address +* isValueOwner -> boolean to match the given address ## Asset - NFT count * fetch count of NFTs owned by asset holding address ✅ diff --git a/docs/nft/OwnerUpdates.md b/docs/nft/OwnerUpdates.md new file mode 100644 index 00000000..2060ba63 --- /dev/null +++ b/docs/nft/OwnerUpdates.md @@ -0,0 +1 @@ +Track when a scope owner/value owner/data access is initiated or updated. diff --git a/gradle.properties b/gradle.properties index fecc5055..fdf9bf8b 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,3 +1,4 @@ kotlin.code.style=official kapt.use.worker.api=false kapt.incremental.apt=false +org.gradle.jvmargs=-Xmx4096M diff --git a/proto/README.md b/proto/README.md index 7b48e972..ad5c355f 100644 --- a/proto/README.md +++ b/proto/README.md @@ -30,8 +30,8 @@ in the `./buildSrc/src/main/kotlin/Dependencies.kt` file: ```kotlin //external protos - const val Provenance = "v1.7.0" - const val Cosmos = "v0.44.0" + const val Provenance = "v1.7.5" + const val Cosmos = "v0.44.3" const val Wasmd = "v0.19.0" const val Ibc = "v1.1.0" ``` @@ -39,7 +39,7 @@ in the `./buildSrc/src/main/kotlin/Dependencies.kt` file: To manually specify the versions run this `gradle` task *from the root project directory*: ```bash -./gradlew downloadProtos --provenance-version v1.7.0 --cosmos-version v0.44.0 --wasmd-version v0.19.0 --ibc-version v1.1.0 +./gradlew downloadProtos --provenance-version v1.7.5 --cosmos-version v0.44.3 --wasmd-version v0.19.0 --ibc-version v1.1.0 ``` > The proto download process does not need to be run very often, @@ -51,5 +51,5 @@ To manually specify the versions run this `gradle` task *from the root project Once the protos have been downloaded, run the `gradle` task *from the root project directory*: ```bash -./gradlew clean proto:build +./gradlew clean proto:generateProto ``` diff --git a/proto/build.gradle.kts b/proto/build.gradle.kts index 97853737..81d5872c 100644 --- a/proto/build.gradle.kts +++ b/proto/build.gradle.kts @@ -7,13 +7,15 @@ import com.google.protobuf.gradle.protoc sourceSets.main { proto.srcDirs("../third_party/proto/") - java.srcDirs("build/generated/source/proto/main/java") } dependencies { - api(Libraries.ProtobufJava) - api(Libraries.GrpcStub) +// protobuf(files("cosmWasm-v0.17.0.tar.gz")) + api(Libraries.ProtobufJavaUtil) + implementation(Libraries.ProtobufKotlin) + api(Libraries.GrpcKotlinStub) api(Libraries.GrpcProtobuf) + implementation(Libraries.GrpcStub) if (JavaVersion.current().isJava9Compatible) { // Workaround for @javax.annotation.Generated @@ -22,6 +24,12 @@ dependencies { } } +tasks.withType().all { + kotlinOptions { + freeCompilerArgs = listOf("-Xopt-in=kotlin.RequiresOptIn") + } +} + protobuf { protoc { // The artifact spec for the Protobuf Compiler @@ -34,12 +42,18 @@ protobuf { id(PluginIds.Grpc) { artifact = Libraries.GrpcArtifact } + id(PluginIds.GrpcKt) { + artifact = Libraries.GrpcKotlinArtifact + } } generateProtoTasks { - ofSourceSet("main").forEach { + all().forEach { it.plugins { - // Apply the "grpc" plugin whose spec is defined above, without options. id(PluginIds.Grpc) + id(PluginIds.GrpcKt) + } + it.builtins { + id(PluginIds.Kotlin) } } } @@ -51,3 +65,13 @@ tasks.register("downloadProtos") { wasmdVersion = Versions.Wasmd ibcVersion = Versions.Ibc } + +//tasks.register("downloadTest"){ +// mapOf("cosmWasm-v0.17.0.tar.gz" to "https://github.com/CosmWasm/wasmd/tarball/v0.17.0") +// .forEach { (k, v) -> download(v,k) } +//} +// +//fun download(url : String, path : String){ +// val destFile = File(path) +// ant.invokeMethod("get", mapOf("src" to url, "dest" to destFile)) +//} diff --git a/service/build.gradle.kts b/service/build.gradle.kts index a18080e2..feeb6495 100644 --- a/service/build.gradle.kts +++ b/service/build.gradle.kts @@ -17,15 +17,6 @@ sourceSets { resources.srcDirs("src/test/resources") } } -// create("integrationTest") { -// withConvention(KotlinSourceSet::class) { -// kotlin.srcDirs("src/test/kotlin") -// compileClasspath += sourceSets.main.get().output + sourceSets.test.get().output + -// configurations.testRuntimeOnly + configurations.testImplementation -// runtimeClasspath += output + compileClasspath + test.get().output -// resources.srcDirs(file("src/test/resources")) -// } -// } } configurations.all { @@ -45,12 +36,12 @@ dependencies { api(Libraries.BouncyCastle) api(Libraries.KotlinXCoRoutinesCore) + api(Libraries.KotlinXCoRoutinesGuava) api(Libraries.ApacheCommonsText) api(Libraries.Khttp) implementation(Libraries.KaseChange) implementation(Libraries.GrpcNetty) - implementation(Libraries.GrpcStart) api(Libraries.LogbackCore) api(Libraries.LogbackClassic) diff --git a/service/src/main/kotlin/io/provenance/explorer/config/GrpcClientConfig.kt b/service/src/main/kotlin/io/provenance/explorer/config/GrpcClientConfig.kt index dab83b9e..61b59c8c 100644 --- a/service/src/main/kotlin/io/provenance/explorer/config/GrpcClientConfig.kt +++ b/service/src/main/kotlin/io/provenance/explorer/config/GrpcClientConfig.kt @@ -1,5 +1,6 @@ package io.provenance.explorer.config +import kotlinx.coroutines.sync.Semaphore import org.springframework.boot.context.properties.EnableConfigurationProperties import org.springframework.context.annotation.Bean import org.springframework.context.annotation.Configuration @@ -13,4 +14,7 @@ class GrpcClientConfig(val props: ExplorerProperties) { @Bean fun channelUri() = URI(props.pbUrl) + + @Bean + fun semaphore() = Semaphore(System.getenv("GRPC_CONCURRENCY")?.toInt() ?: 20) } diff --git a/service/src/main/kotlin/io/provenance/explorer/config/GrpcLoggingInterceptor.kt b/service/src/main/kotlin/io/provenance/explorer/config/GrpcLoggingInterceptor.kt index 47910222..520f2d86 100644 --- a/service/src/main/kotlin/io/provenance/explorer/config/GrpcLoggingInterceptor.kt +++ b/service/src/main/kotlin/io/provenance/explorer/config/GrpcLoggingInterceptor.kt @@ -11,7 +11,7 @@ import org.springframework.stereotype.Component import java.util.concurrent.TimeUnit @Component -class GrpcLoggingInterceptor : ClientInterceptor { +class GrpcLoggingInterceptor() : ClientInterceptor { private val logger = logger() override fun interceptCall( method: MethodDescriptor, @@ -20,7 +20,7 @@ class GrpcLoggingInterceptor : ClientInterceptor { ): ClientCall { return object : BackendForwardingClientCall( method, - next.newCall(method, callOptions.withDeadlineAfter(10000, TimeUnit.MILLISECONDS)) + next.newCall(method, callOptions.withDeadlineAfter(60, TimeUnit.SECONDS)) ) { override fun sendMessage(message: M) { logger.debug("Requesting external api method: $methodName") diff --git a/service/src/main/kotlin/io/provenance/explorer/domain/core/MetadataAddress.kt b/service/src/main/kotlin/io/provenance/explorer/domain/core/MetadataAddress.kt index b396881c..1ee0b820 100644 --- a/service/src/main/kotlin/io/provenance/explorer/domain/core/MetadataAddress.kt +++ b/service/src/main/kotlin/io/provenance/explorer/domain/core/MetadataAddress.kt @@ -183,3 +183,8 @@ fun String.toMAddressContractSpec() = MetadataAddress.forContractSpecification(t fun UUID.toMAddressContractSpec() = MetadataAddress.forContractSpecification(this) fun String.blankToNull() = this.ifBlank { null } + +fun String.isMAddress() = + this.startsWith(PREFIX_SCOPE_SPECIFICATION) || this.startsWith(PREFIX_SCOPE) || + this.startsWith(PREFIX_CONTRACT_SPECIFICATION) || this.startsWith(PREFIX_SESSION) || + this.startsWith(PREFIX_RECORD_SPECIFICATION) || this.startsWith(PREFIX_RECORD) diff --git a/service/src/main/kotlin/io/provenance/explorer/domain/entities/Blocks.kt b/service/src/main/kotlin/io/provenance/explorer/domain/entities/Blocks.kt index 1ef7bd97..cac8e7e1 100644 --- a/service/src/main/kotlin/io/provenance/explorer/domain/entities/Blocks.kt +++ b/service/src/main/kotlin/io/provenance/explorer/domain/entities/Blocks.kt @@ -9,14 +9,18 @@ import io.provenance.explorer.domain.core.sql.ExtractDay import io.provenance.explorer.domain.core.sql.ExtractHour import io.provenance.explorer.domain.core.sql.jsonb import io.provenance.explorer.domain.extensions.average +import io.provenance.explorer.domain.extensions.exec +import io.provenance.explorer.domain.extensions.map import io.provenance.explorer.domain.extensions.startOfDay import io.provenance.explorer.domain.models.explorer.DateTruncGranularity +import io.provenance.explorer.domain.models.explorer.MissedBlockPeriod import io.provenance.explorer.domain.models.explorer.TxHeatmap import io.provenance.explorer.domain.models.explorer.TxHeatmapDay import io.provenance.explorer.domain.models.explorer.TxHeatmapHour import io.provenance.explorer.domain.models.explorer.TxHeatmapRaw import io.provenance.explorer.domain.models.explorer.TxHeatmapRes import io.provenance.explorer.domain.models.explorer.TxHistory +import io.provenance.explorer.domain.models.explorer.ValidatorMoniker import org.jetbrains.exposed.dao.Entity import org.jetbrains.exposed.dao.EntityClass import org.jetbrains.exposed.dao.IntEntity @@ -24,11 +28,12 @@ import org.jetbrains.exposed.dao.IntEntityClass import org.jetbrains.exposed.dao.id.EntityID import org.jetbrains.exposed.dao.id.IdTable import org.jetbrains.exposed.dao.id.IntIdTable +import org.jetbrains.exposed.sql.ColumnType import org.jetbrains.exposed.sql.IntegerColumnType import org.jetbrains.exposed.sql.Max import org.jetbrains.exposed.sql.SortOrder -import org.jetbrains.exposed.sql.SqlExpressionBuilder.minus import org.jetbrains.exposed.sql.Sum +import org.jetbrains.exposed.sql.VarCharColumnType import org.jetbrains.exposed.sql.and import org.jetbrains.exposed.sql.andWhere import org.jetbrains.exposed.sql.count @@ -232,6 +237,30 @@ object MissedBlocksTable : IntIdTable(name = "missed_blocks") { class MissedBlocksRecord(id: EntityID) : IntEntity(id) { companion object : IntEntityClass(MissedBlocksTable) { + fun findValidatorsWithMissedBlocksForPeriod(fromHeight: Int, toHeight: Int, valConsAddr: String?) = transaction { + val query = "SELECT * FROM missed_block_periods(?, ?, ?) " + val arguments = mutableListOf>( + Pair(IntegerColumnType(), fromHeight), + Pair(IntegerColumnType(), toHeight), + Pair(VarCharColumnType(128), valConsAddr) + ) + query.exec(arguments).map { + if (it.getString("val_cons_address") != null) + MissedBlockPeriod( + ValidatorMoniker(it.getString("val_cons_address"), null, null), + (it.getArray("blocks").array as Array).toList() + ) + else null + }.toMutableList().mapNotNull { it } + } + + fun findDistinctValidatorsWithMissedBlocksForPeriod(fromHeight: Int, toHeight: Int) = transaction { + val query = "SELECT distinct val_cons_address FROM missed_block_periods(?, ?, NULL);" + val arguments = listOf(Pair(IntegerColumnType(), fromHeight), Pair(IntegerColumnType(), toHeight)) + + query.exec(arguments).map { it.getString("val_cons_address") } + } + fun findLatestForVal(valconsAddr: String) = transaction { MissedBlocksRecord.find { MissedBlocksTable.valConsAddr eq valconsAddr } .orderBy(Pair(MissedBlocksTable.blockHeight, SortOrder.DESC)) diff --git a/service/src/main/kotlin/io/provenance/explorer/domain/entities/Governance.kt b/service/src/main/kotlin/io/provenance/explorer/domain/entities/Governance.kt index ddc99782..b94f2f71 100644 --- a/service/src/main/kotlin/io/provenance/explorer/domain/entities/Governance.kt +++ b/service/src/main/kotlin/io/provenance/explorer/domain/entities/Governance.kt @@ -18,8 +18,8 @@ import org.jetbrains.exposed.dao.id.IntIdTable import org.jetbrains.exposed.sql.SortOrder import org.jetbrains.exposed.sql.and import org.jetbrains.exposed.sql.innerJoin -import org.jetbrains.exposed.sql.insert import org.jetbrains.exposed.sql.insertAndGetId +import org.jetbrains.exposed.sql.insertIgnore import org.jetbrains.exposed.sql.jodatime.datetime import org.jetbrains.exposed.sql.select import org.jetbrains.exposed.sql.transactions.transaction @@ -329,7 +329,7 @@ class ProposalMonitorRecord(id: EntityID) : IntEntity(id) { proposalType: ProposalType, dataHash: String ) = transaction { - ProposalMonitorTable.insert { + ProposalMonitorTable.insertIgnore { it[this.proposalId] = proposalId it[this.submittedHeight] = submittedHeight it[this.proposedCompletionHeight] = proposedCompletionHeight diff --git a/service/src/main/kotlin/io/provenance/explorer/domain/entities/Nfts.kt b/service/src/main/kotlin/io/provenance/explorer/domain/entities/Nfts.kt index ae7f3c06..49dcf82f 100644 --- a/service/src/main/kotlin/io/provenance/explorer/domain/entities/Nfts.kt +++ b/service/src/main/kotlin/io/provenance/explorer/domain/entities/Nfts.kt @@ -1,9 +1,12 @@ package io.provenance.explorer.domain.entities +import io.provenance.explorer.domain.models.explorer.NftVOTransferObj import org.jetbrains.exposed.dao.IntEntity import org.jetbrains.exposed.dao.IntEntityClass import org.jetbrains.exposed.dao.id.EntityID import org.jetbrains.exposed.dao.id.IntIdTable +import org.jetbrains.exposed.sql.SortOrder +import org.jetbrains.exposed.sql.and import org.jetbrains.exposed.sql.insertAndGetId import org.jetbrains.exposed.sql.transactions.transaction @@ -103,3 +106,61 @@ class NftContractSpecRecord(id: EntityID) : IntEntity(id) { var address by NftContractSpecTable.address var deleted by NftContractSpecTable.deleted } + +enum class ScopeTransferType { + VALUE_OWNER_INITIAL, + VALUE_OWNER_TRANSFER +} + +object NftScopeVOTransferTable : IntIdTable(name = "nft_scope_value_owner_transfer") { + val scopeId = integer("scope_id") + val scopeAddr = varchar("scope_addr", 128) + val address = varchar("address", 256) + val txId = reference("tx_id", TxCacheTable) + val blockHeight = integer("block_height") + val txHash = varchar("tx_hash", 64) +} + +class NftScopeVOTransferRecord(id: EntityID) : IntEntity(id) { + companion object : IntEntityClass(NftScopeVOTransferTable) { + + fun findByUniqueKey(scopeId: Int, address: String, txId: Int) = transaction { + NftScopeVOTransferRecord.find { + (NftScopeVOTransferTable.scopeId eq scopeId) and + (NftScopeVOTransferTable.address eq address) and + (NftScopeVOTransferTable.txId eq txId) + } + .firstOrNull() + } + + fun getOrInsert(transferObj: NftVOTransferObj) = + transaction { + findByUniqueKey( + transferObj.scope.id.value, + transferObj.address, + transferObj.tx.id.value + ) + ?: NftScopeVOTransferTable.insertAndGetId { + it[this.scopeId] = transferObj.scope.id.value + it[this.scopeAddr] = transferObj.scope.address + it[this.address] = transferObj.address + it[this.txId] = transferObj.tx.id + it[this.blockHeight] = transferObj.tx.height + it[this.txHash] = transferObj.tx.hash + }.let { findById(it)!! } + } + + fun findByScopeAddr(addr: String) = transaction { + NftScopeVOTransferRecord.find { NftScopeVOTransferTable.scopeAddr eq addr } + .orderBy(Pair(NftScopeVOTransferTable.blockHeight, SortOrder.ASC)) + .toList() + } + } + + var scopeId by NftScopeVOTransferTable.scopeId + var scopeAddr by NftScopeVOTransferTable.scopeAddr + var address by NftScopeVOTransferTable.address + var txId by TxCacheRecord referencedOn NftScopeVOTransferTable.txId + var blockHeight by NftScopeVOTransferTable.blockHeight + var txHash by NftScopeVOTransferTable.txHash +} diff --git a/service/src/main/kotlin/io/provenance/explorer/domain/entities/SmartContracts.kt b/service/src/main/kotlin/io/provenance/explorer/domain/entities/SmartContracts.kt index 0355d3e6..2d249995 100644 --- a/service/src/main/kotlin/io/provenance/explorer/domain/entities/SmartContracts.kt +++ b/service/src/main/kotlin/io/provenance/explorer/domain/entities/SmartContracts.kt @@ -45,6 +45,14 @@ class SmCodeRecord(id: EntityID) : IntEntity(id) { it[this.data] = data }.value } + + fun getPaginated(offset: Int, limit: Int) = transaction { + SmCodeRecord + .all() + .orderBy(Pair(SmCodeTable.id, SortOrder.DESC)) + .limit(limit, offset.toLong()) + .toList() + } } var creationHeight by SmCodeTable.creationHeight diff --git a/service/src/main/kotlin/io/provenance/explorer/domain/entities/Validators.kt b/service/src/main/kotlin/io/provenance/explorer/domain/entities/Validators.kt index ef1db690..7b5dbf75 100644 --- a/service/src/main/kotlin/io/provenance/explorer/domain/entities/Validators.kt +++ b/service/src/main/kotlin/io/provenance/explorer/domain/entities/Validators.kt @@ -65,7 +65,10 @@ object StakingValidatorCacheTable : IntIdTable(name = "staking_validator_cache") class StakingValidatorCacheRecord(id: EntityID) : IntEntity(id) { companion object : IntEntityClass(StakingValidatorCacheTable) { - val logger = logger(StakingValidatorCacheRecord::class) + fun findByOperAddr(operAddr: String) = transaction { + StakingValidatorCacheRecord.find { StakingValidatorCacheTable.operatorAddress eq operAddr } + .firstOrNull() + } fun insertIgnore( operator: String, @@ -138,7 +141,7 @@ class ValidatorStateRecord(id: EntityID) : IntEntity(id) { if (ids.isNotEmpty()) { val arguments = ids.joinToString(", ", "(", ")") val query = "SELECT * FROM current_validator_state WHERE operator_addr_id in $arguments".trimIndent() - query.execAndMap() { it.toCurrentValidatorState() } + query.execAndMap { it.toCurrentValidatorState() } } else listOf() } @@ -154,6 +157,12 @@ class ValidatorStateRecord(id: EntityID) : IntEntity(id) { query.execAndMap(arguments) { it.toCurrentValidatorState() }.firstOrNull() } + fun findByConsensusAddressIn(addresses: List) = transaction { + val arguments = addresses.joinToString("', '", "('", "')") + val query = "SELECT * FROM current_validator_state WHERE consensus_address IN $arguments".trimIndent() + query.execAndMap { it.toCurrentValidatorState() } + } + fun findByOperator(address: String) = transaction { val query = "SELECT * FROM current_validator_state WHERE operator_address = ?".trimIndent() val arguments = listOf(Pair(VarCharColumnType(128), address)) diff --git a/service/src/main/kotlin/io/provenance/explorer/domain/extensions/MessageTranslationExtensions.kt b/service/src/main/kotlin/io/provenance/explorer/domain/extensions/MessageTranslationExtensions.kt index 31caf29e..2db358e8 100644 --- a/service/src/main/kotlin/io/provenance/explorer/domain/extensions/MessageTranslationExtensions.kt +++ b/service/src/main/kotlin/io/provenance/explorer/domain/extensions/MessageTranslationExtensions.kt @@ -1,13 +1,15 @@ package io.provenance.explorer.domain.extensions import com.fasterxml.jackson.databind.JsonNode +import com.fasterxml.jackson.databind.node.ArrayNode import com.fasterxml.jackson.databind.node.ObjectNode import com.google.protobuf.Message import com.google.protobuf.util.JsonFormat import io.provenance.explorer.JSON_NODE_FACTORY import io.provenance.explorer.OBJECT_MAPPER +import io.provenance.explorer.domain.core.isMAddress -val protoTypesToCheckForMetadata = arrayOf( +val protoTypesToCheckForMetadata = listOf( "/provenance.metadata.v1.MsgWriteScopeRequest", "/provenance.metadata.v1.MsgDeleteScopeRequest", "/provenance.metadata.v1.MsgWriteRecordSpecificationRequest", @@ -27,16 +29,20 @@ val protoTypesToCheckForMetadata = arrayOf( "/provenance.metadata.v1.MsgDeleteContractSpecFromScopeSpecRequest" ) -val protoTypesFieldsToCheckForMetadata = arrayOf( +val protoTypesFieldsToCheckForMetadata = listOf( "scopeId", "specificationId", "recordId", "sessionId", "contractSpecificationId", - "scopeSpecificationId" + "scopeSpecificationId", + "contractSpecIds", + "scopeSpecId", + "contractSpecId", + "recordSpecId" ) -val protoTypesToCheckForSmartContract = arrayOf( +val protoTypesToCheckForSmartContract = listOf( "/cosmwasm.wasm.v1beta1.MsgInstantiateContract", "/cosmwasm.wasm.v1beta1.MsgExecuteContract", "/cosmwasm.wasm.v1beta1.MsgMigrateContract", @@ -45,7 +51,7 @@ val protoTypesToCheckForSmartContract = arrayOf( "/cosmwasm.wasm.v1.MsgMigrateContract" ) -val protoTypesFieldsToCheckForSmartContract = arrayOf( +val protoTypesFieldsToCheckForSmartContract = listOf( "initMsg", "msg", "migrateMsg" @@ -71,6 +77,13 @@ fun Message.toObjectNodeNonTxMsg(protoPrinter: JsonFormat.Printer, fieldNames: L node } +fun Message.toObjectNodeMAddressValues(protoPrinter: JsonFormat.Printer, fieldNames: List) = + OBJECT_MAPPER.readTree(protoPrinter.print(this)) + .let { node -> + fieldNames.forEach { fromBase64ToMAddress(node, it) } + node + } + fun fromBase64ToObject(jsonNode: JsonNode, fieldName: String) { var found = false @@ -89,9 +102,26 @@ fun fromBase64ToMAddress(jsonNode: JsonNode, fieldName: String) { var found = false if (jsonNode.has(fieldName)) { - val newValue = jsonNode.get(fieldName).asText().fromBase64ToMAddress().toString() - (jsonNode as ObjectNode).replace(fieldName, JSON_NODE_FACTORY.textNode(newValue)) - found = true // stop after first find + when { + jsonNode.get(fieldName).isTextual -> { + val value = jsonNode.get(fieldName).asText() + if (!value.isMAddress()) + (jsonNode as ObjectNode).replace( + fieldName, + JSON_NODE_FACTORY.textNode(value.fromBase64ToMAddress().toString()) + ) + } + jsonNode.get(fieldName).isArray -> { + val oldArray = (jsonNode.get(fieldName) as ArrayNode) + val newArray = JSON_NODE_FACTORY.arrayNode() + oldArray.forEach { + val value = it.asText().fromBase64ToMAddress().toString() + newArray.add(JSON_NODE_FACTORY.textNode(value)) + } + (jsonNode as ObjectNode).replace(fieldName, newArray) + } + } + found = true } if (!found) { diff --git a/service/src/main/kotlin/io/provenance/explorer/domain/models/explorer/AssetModels.kt b/service/src/main/kotlin/io/provenance/explorer/domain/models/explorer/AssetModels.kt index 2150dc3b..0cd63198 100644 --- a/service/src/main/kotlin/io/provenance/explorer/domain/models/explorer/AssetModels.kt +++ b/service/src/main/kotlin/io/provenance/explorer/domain/models/explorer/AssetModels.kt @@ -46,7 +46,8 @@ data class AccountDetail( val publicKeys: Signatures, val accountName: String?, val attributes: List, - val tokens: TokenCounts + val tokens: TokenCounts, + val isContract: Boolean ) data class AssetManagement( diff --git a/service/src/main/kotlin/io/provenance/explorer/domain/models/explorer/CommonModels.kt b/service/src/main/kotlin/io/provenance/explorer/domain/models/explorer/CommonModels.kt index 60655ad0..6031c0d7 100644 --- a/service/src/main/kotlin/io/provenance/explorer/domain/models/explorer/CommonModels.kt +++ b/service/src/main/kotlin/io/provenance/explorer/domain/models/explorer/CommonModels.kt @@ -54,3 +54,6 @@ data class TokenDistribution( val amount: TokenDistributionAmount, val percent: String ) + +enum class Timeframe { WEEK, DAY, HOUR, FOREVER } +const val hourlyBlockCount = 720 diff --git a/service/src/main/kotlin/io/provenance/explorer/domain/models/explorer/NftModels.kt b/service/src/main/kotlin/io/provenance/explorer/domain/models/explorer/NftModels.kt index d92cfe1b..d64df8d5 100644 --- a/service/src/main/kotlin/io/provenance/explorer/domain/models/explorer/NftModels.kt +++ b/service/src/main/kotlin/io/provenance/explorer/domain/models/explorer/NftModels.kt @@ -1,13 +1,19 @@ package io.provenance.explorer.domain.models.explorer +import io.provenance.explorer.domain.entities.NftScopeRecord +import io.provenance.explorer.domain.entities.TxCacheRecord import io.provenance.metadata.v1.Description import io.provenance.metadata.v1.Party +import io.provenance.metadata.v1.RecordOutput data class ScopeListview( val scopeAddr: String, val specName: String?, val specAddr: String, - val lastUpdated: String + val lastUpdated: String, + val isOwner: Boolean, + val isDataAccess: Boolean, + val isValueOwner: Boolean ) data class ScopeDetail( @@ -16,6 +22,7 @@ data class ScopeDetail( val specAddr: String, val description: SpecDescrip?, val owners: List, + val dataAccess: List, val valueOwner: String? ) @@ -48,11 +55,26 @@ data class RecordDetail( val recordAddr: String, val recordSpecAddr: String, val lastModified: String, - val responsibleParties: List + val responsibleParties: List, + val outputs: List ) +data class RecordInputOutput( + val name: String, + val hash: String, + val status: String +) + +fun RecordOutput.toDataObject(name: String) = RecordInputOutput(name, this.hash, this.status.name) + data class RecordSpecDetail( val contractSpecAddr: String, val recordSpecAddr: String, val responsibleParties: List ) + +data class NftVOTransferObj( + val scope: NftScopeRecord, + val address: String, + val tx: TxCacheRecord +) diff --git a/service/src/main/kotlin/io/provenance/explorer/domain/models/explorer/SmartContractModels.kt b/service/src/main/kotlin/io/provenance/explorer/domain/models/explorer/SmartContractModels.kt new file mode 100644 index 00000000..07fa8764 --- /dev/null +++ b/service/src/main/kotlin/io/provenance/explorer/domain/models/explorer/SmartContractModels.kt @@ -0,0 +1,17 @@ +package io.provenance.explorer.domain.models.explorer + +data class Contract( + val contractAddress: String, + val creationHeight: Int, + val codeId: Int, + val creator: String, + val admin: String?, + val label: String? +) + +data class Code( + val codeId: Int, + val creationHeight: Int, + val creator: String?, + val dataHash: String? +) diff --git a/service/src/main/kotlin/io/provenance/explorer/domain/models/explorer/ValidatorModels.kt b/service/src/main/kotlin/io/provenance/explorer/domain/models/explorer/ValidatorModels.kt index 42ac967b..616c10eb 100644 --- a/service/src/main/kotlin/io/provenance/explorer/domain/models/explorer/ValidatorModels.kt +++ b/service/src/main/kotlin/io/provenance/explorer/domain/models/explorer/ValidatorModels.kt @@ -99,3 +99,31 @@ data class ValidatorAtHeight( val isProposer: Boolean = false, val didVote: Boolean = true ) + +data class MissedBlocksTimeframe( + val fromHeight: Int, + val toHeight: Int, + val addresses: List +) + +data class ValidatorMissedBlocks( + val validator: ValidatorMoniker, + val missedBlocks: List = listOf() +) + +data class MissedBlockSet( + val min: Int, + val max: Int, + val count: Int +) + +data class ValidatorMoniker( + val valConsAddress: String, + var operatorAddr: String?, + var moniker: String? +) + +data class MissedBlockPeriod( + val validator: ValidatorMoniker, + val blocks: List +) diff --git a/service/src/main/kotlin/io/provenance/explorer/grpc/extensions/Domain.kt b/service/src/main/kotlin/io/provenance/explorer/grpc/extensions/Domain.kt index 88b605fd..87b7c191 100644 --- a/service/src/main/kotlin/io/provenance/explorer/grpc/extensions/Domain.kt +++ b/service/src/main/kotlin/io/provenance/explorer/grpc/extensions/Domain.kt @@ -3,6 +3,7 @@ package io.provenance.explorer.grpc.extensions import com.google.protobuf.Any import cosmos.auth.v1beta1.Auth import cosmos.base.query.v1beta1.Pagination +import cosmos.base.query.v1beta1.pageRequest import cosmos.crypto.multisig.Keys import io.provenance.explorer.config.ResourceNotFoundException import io.provenance.explorer.domain.core.logger @@ -80,3 +81,19 @@ fun getEscrowAccountAddress(portId: String, channelId: String, hrpPrefix: String fun getPaginationBuilder(offset: Int, limit: Int) = Pagination.PageRequest.newBuilder().setOffset(offset.toLong()).setLimit(limit.toLong()).setCountTotal(true) + +fun getPaginationBuilderNoCount(offset: Int, limit: Int) = + Pagination.PageRequest.newBuilder().setOffset(offset.toLong()).setLimit(limit.toLong()) + +fun getPagination(offset: Int, limit: Int) = + pageRequest { + this.offset = offset.toLong() + this.limit = limit.toLong() + this.countTotal = true + } + +fun getPaginationNoCount(offset: Int, limit: Int) = + pageRequest { + this.offset = offset.toLong() + this.limit = limit.toLong() + } diff --git a/service/src/main/kotlin/io/provenance/explorer/grpc/v1/AccountGrpcClient.kt b/service/src/main/kotlin/io/provenance/explorer/grpc/v1/AccountGrpcClient.kt index e48f4f68..1a739a01 100644 --- a/service/src/main/kotlin/io/provenance/explorer/grpc/v1/AccountGrpcClient.kt +++ b/service/src/main/kotlin/io/provenance/explorer/grpc/v1/AccountGrpcClient.kt @@ -1,30 +1,37 @@ package io.provenance.explorer.grpc.v1 +import cosmos.auth.v1beta1.queryAccountRequest +import cosmos.bank.v1beta1.queryAllBalancesRequest +import cosmos.bank.v1beta1.queryDenomMetadataRequest +import cosmos.bank.v1beta1.queryDenomMetadataResponse +import cosmos.bank.v1beta1.queryDenomsMetadataRequest +import cosmos.bank.v1beta1.queryParamsRequest +import cosmos.bank.v1beta1.querySupplyOfRequest +import cosmos.distribution.v1beta1.queryDelegationTotalRewardsRequest +import cosmos.staking.v1beta1.queryDelegatorDelegationsRequest +import cosmos.staking.v1beta1.queryDelegatorDelegationsResponse +import cosmos.staking.v1beta1.queryDelegatorUnbondingDelegationsRequest +import cosmos.staking.v1beta1.queryRedelegationsRequest import io.grpc.ManagedChannelBuilder import io.provenance.explorer.config.GrpcLoggingInterceptor -import io.provenance.explorer.grpc.extensions.getPaginationBuilder +import io.provenance.explorer.grpc.extensions.getPagination import org.springframework.stereotype.Component import java.net.URI import java.util.concurrent.TimeUnit -import cosmos.auth.v1beta1.QueryGrpc as AuthQueryGrpc -import cosmos.auth.v1beta1.QueryOuterClass as AuthOuterClass -import cosmos.bank.v1beta1.QueryGrpc as BankQueryGrpc -import cosmos.bank.v1beta1.QueryOuterClass as BankOuterClass -import cosmos.distribution.v1beta1.QueryGrpc as DistGrpc -import cosmos.distribution.v1beta1.QueryOuterClass as DistOuterClass -import cosmos.mint.v1beta1.QueryGrpc as MintGrpc -import cosmos.mint.v1beta1.QueryOuterClass as MintOuterClass -import cosmos.staking.v1beta1.QueryGrpc as StakingGrpc -import cosmos.staking.v1beta1.QueryOuterClass as StakingOuterClass +import cosmos.auth.v1beta1.QueryGrpcKt as AuthQueryGrpc +import cosmos.bank.v1beta1.QueryGrpcKt as BankQueryGrpc +import cosmos.distribution.v1beta1.QueryGrpcKt as DistGrpc +import cosmos.mint.v1beta1.QueryGrpcKt as MintGrpc +import cosmos.staking.v1beta1.QueryGrpcKt as StakingGrpc @Component class AccountGrpcClient(channelUri: URI) { - private val authClient: AuthQueryGrpc.QueryBlockingStub - private val bankClient: BankQueryGrpc.QueryBlockingStub - private val stakingClient: StakingGrpc.QueryBlockingStub - private val distClient: DistGrpc.QueryBlockingStub - private val mintClient: MintGrpc.QueryBlockingStub + private val authClient: AuthQueryGrpc.QueryCoroutineStub + private val bankClient: BankQueryGrpc.QueryCoroutineStub + private val stakingClient: StakingGrpc.QueryCoroutineStub + private val distClient: DistGrpc.QueryCoroutineStub + private val mintClient: MintGrpc.QueryCoroutineStub init { val channel = @@ -36,89 +43,81 @@ class AccountGrpcClient(channelUri: URI) { it.usePlaintext() } } - .idleTimeout(60, TimeUnit.SECONDS) - .keepAliveTime(10, TimeUnit.SECONDS) - .keepAliveTimeout(10, TimeUnit.SECONDS) + .idleTimeout(5, TimeUnit.MINUTES) + .keepAliveTime(60, TimeUnit.SECONDS) + .keepAliveTimeout(20, TimeUnit.SECONDS) .intercept(GrpcLoggingInterceptor()) .build() - authClient = AuthQueryGrpc.newBlockingStub(channel) - bankClient = BankQueryGrpc.newBlockingStub(channel) - stakingClient = StakingGrpc.newBlockingStub(channel) - distClient = DistGrpc.newBlockingStub(channel) - mintClient = MintGrpc.newBlockingStub(channel) + authClient = AuthQueryGrpc.QueryCoroutineStub(channel) + bankClient = BankQueryGrpc.QueryCoroutineStub(channel) + stakingClient = StakingGrpc.QueryCoroutineStub(channel) + distClient = DistGrpc.QueryCoroutineStub(channel) + mintClient = MintGrpc.QueryCoroutineStub(channel) } - fun getAccountInfo(address: String) = + suspend fun getAccountInfo(address: String) = try { - authClient.account(AuthOuterClass.QueryAccountRequest.newBuilder().setAddress(address).build()).account + authClient.account(queryAccountRequest { this.address = address }).account } catch (e: Exception) { null } - fun getAccountBalances(address: String, offset: Int, limit: Int) = + suspend fun getAccountBalances(address: String, offset: Int, limit: Int) = bankClient.allBalances( - BankOuterClass.QueryAllBalancesRequest.newBuilder() - .setAddress(address) - .setPagination(getPaginationBuilder(offset, limit)) - .build() + queryAllBalancesRequest { + this.address = address + this.pagination = getPagination(offset, limit) + } ) - fun getCurrentSupply(denom: String) = - bankClient.supplyOf(BankOuterClass.QuerySupplyOfRequest.newBuilder().setDenom(denom).build()).amount + suspend fun getCurrentSupply(denom: String) = + bankClient.supplyOf(querySupplyOfRequest { this.denom = denom }).amount - fun getDenomMetadata(denom: String) = + suspend fun getDenomMetadata(denom: String) = try { - bankClient.denomMetadata(BankOuterClass.QueryDenomMetadataRequest.newBuilder().setDenom(denom).build()) + bankClient.denomMetadata(queryDenomMetadataRequest { this.denom = denom }) } catch (e: Exception) { - BankOuterClass.QueryDenomMetadataResponse.getDefaultInstance() + queryDenomMetadataResponse { } } - fun getAllDenomMetadata() = - bankClient.denomsMetadata( - BankOuterClass.QueryDenomsMetadataRequest.newBuilder() - .setPagination(getPaginationBuilder(0, 100)) - .build() - ) + suspend fun getAllDenomMetadata() = + bankClient.denomsMetadata(queryDenomsMetadataRequest { this.pagination = getPagination(0, 100) }) - fun getDelegations(address: String, offset: Int, limit: Int) = + suspend fun getDelegations(address: String, offset: Int, limit: Int) = try { stakingClient.delegatorDelegations( - StakingOuterClass.QueryDelegatorDelegationsRequest.newBuilder() - .setDelegatorAddr(address) - .setPagination(getPaginationBuilder(offset, limit)) - .build() + queryDelegatorDelegationsRequest { + this.delegatorAddr = address + this.pagination = getPagination(offset, limit) + } ) } catch (e: Exception) { - StakingOuterClass.QueryDelegatorDelegationsResponse.getDefaultInstance() + queryDelegatorDelegationsResponse { } } - fun getUnbondingDelegations(address: String, offset: Int, limit: Int) = + suspend fun getUnbondingDelegations(address: String, offset: Int, limit: Int) = stakingClient.delegatorUnbondingDelegations( - StakingOuterClass.QueryDelegatorUnbondingDelegationsRequest.newBuilder() - .setDelegatorAddr(address) - .setPagination(getPaginationBuilder(offset, limit)) - .build() + queryDelegatorUnbondingDelegationsRequest { + this.delegatorAddr = address + this.pagination = getPagination(offset, limit) + } ) - fun getRedelegations(address: String, offset: Int, limit: Int) = + suspend fun getRedelegations(address: String, offset: Int, limit: Int) = stakingClient.redelegations( - StakingOuterClass.QueryRedelegationsRequest.newBuilder() - .setDelegatorAddr(address) - .setPagination(getPaginationBuilder(offset, limit)) - .build() + queryRedelegationsRequest { + this.delegatorAddr = address + this.pagination = getPagination(offset, limit) + } ) - fun getRewards(delAddr: String) = - distClient.delegationTotalRewards( - DistOuterClass.QueryDelegationTotalRewardsRequest.newBuilder() - .setDelegatorAddress(delAddr) - .build() - ) + suspend fun getRewards(delAddr: String) = + distClient.delegationTotalRewards(queryDelegationTotalRewardsRequest { this.delegatorAddress = delAddr }) - fun getBankParams() = bankClient.params(BankOuterClass.QueryParamsRequest.newBuilder().build()) + suspend fun getBankParams() = bankClient.params(queryParamsRequest { }) - fun getAuthParams() = authClient.params(AuthOuterClass.QueryParamsRequest.newBuilder().build()) + suspend fun getAuthParams() = authClient.params(cosmos.auth.v1beta1.queryParamsRequest { }) - fun getMintParams() = mintClient.params(MintOuterClass.QueryParamsRequest.newBuilder().build()) + suspend fun getMintParams() = mintClient.params(cosmos.mint.v1beta1.queryParamsRequest { }) } diff --git a/service/src/main/kotlin/io/provenance/explorer/grpc/v1/AttributeGrpcClient.kt b/service/src/main/kotlin/io/provenance/explorer/grpc/v1/AttributeGrpcClient.kt index 5f614f48..d1206393 100644 --- a/service/src/main/kotlin/io/provenance/explorer/grpc/v1/AttributeGrpcClient.kt +++ b/service/src/main/kotlin/io/provenance/explorer/grpc/v1/AttributeGrpcClient.kt @@ -2,23 +2,22 @@ package io.provenance.explorer.grpc.v1 import io.grpc.ManagedChannelBuilder import io.provenance.attribute.v1.Attribute -import io.provenance.attribute.v1.QueryAttributesRequest +import io.provenance.attribute.v1.queryAttributesRequest +import io.provenance.attribute.v1.queryParamsRequest import io.provenance.explorer.config.GrpcLoggingInterceptor -import io.provenance.explorer.grpc.extensions.getPaginationBuilder -import io.provenance.name.v1.QueryReverseLookupRequest +import io.provenance.explorer.grpc.extensions.getPagination +import io.provenance.name.v1.queryReverseLookupRequest import org.springframework.stereotype.Component import java.net.URI import java.util.concurrent.TimeUnit -import io.provenance.attribute.v1.QueryGrpc as AttrQueryGrpc -import io.provenance.attribute.v1.QueryParamsRequest as AttrRequest -import io.provenance.name.v1.QueryGrpc as NameQueryGrpc -import io.provenance.name.v1.QueryParamsRequest as NameRequest +import io.provenance.attribute.v1.QueryGrpcKt as AttrQueryGrpc +import io.provenance.name.v1.QueryGrpcKt as NameQueryGrpc @Component class AttributeGrpcClient(channelUri: URI) { - private val attrClient: AttrQueryGrpc.QueryBlockingStub - private val nameClient: NameQueryGrpc.QueryBlockingStub + private val attrClient: AttrQueryGrpc.QueryCoroutineStub + private val nameClient: NameQueryGrpc.QueryCoroutineStub init { val channel = @@ -36,20 +35,20 @@ class AttributeGrpcClient(channelUri: URI) { .intercept(GrpcLoggingInterceptor()) .build() - attrClient = AttrQueryGrpc.newBlockingStub(channel) - nameClient = NameQueryGrpc.newBlockingStub(channel) + attrClient = AttrQueryGrpc.QueryCoroutineStub(channel) + nameClient = NameQueryGrpc.QueryCoroutineStub(channel) } - fun getAllAttributesForAddress(address: String?): MutableList { + suspend fun getAllAttributesForAddress(address: String?): MutableList { if (address == null) return mutableListOf() var offset = 0 val limit = 100 val results = attrClient.attributes( - QueryAttributesRequest.newBuilder() - .setAccount(address) - .setPagination(getPaginationBuilder(offset, limit)) - .build() + queryAttributesRequest { + this.account = address + this.pagination = getPagination(offset, limit) + } ) val total = results.pagination?.total ?: results.attributesCount.toLong() @@ -58,10 +57,10 @@ class AttributeGrpcClient(channelUri: URI) { while (attributes.count() < total) { offset += limit attrClient.attributes( - QueryAttributesRequest.newBuilder() - .setAccount(address) - .setPagination(getPaginationBuilder(offset, limit)) - .build() + queryAttributesRequest { + this.account = address + this.pagination = getPagination(offset, limit) + } ) .let { attributes.addAll(it.attributesList) } } @@ -69,14 +68,14 @@ class AttributeGrpcClient(channelUri: URI) { return attributes } - fun getNamesForAddress(address: String, offset: Int, limit: Int) = + suspend fun getNamesForAddress(address: String, offset: Int, limit: Int) = nameClient.reverseLookup( - QueryReverseLookupRequest.newBuilder() - .setAddress(address) - .setPagination(getPaginationBuilder(offset, limit)) - .build() + queryReverseLookupRequest { + this.address = address + this.pagination = getPagination(offset, limit) + } ) - fun getAttrParams() = attrClient.params(AttrRequest.newBuilder().build()) - fun getNameParams() = nameClient.params(NameRequest.newBuilder().build()) + suspend fun getAttrParams() = attrClient.params(queryParamsRequest { }) + suspend fun getNameParams() = nameClient.params(io.provenance.name.v1.queryParamsRequest { }) } diff --git a/service/src/main/kotlin/io/provenance/explorer/grpc/v1/MarkerGrpcClient.kt b/service/src/main/kotlin/io/provenance/explorer/grpc/v1/MarkerGrpcClient.kt index 0ed1d0cb..099f8d53 100644 --- a/service/src/main/kotlin/io/provenance/explorer/grpc/v1/MarkerGrpcClient.kt +++ b/service/src/main/kotlin/io/provenance/explorer/grpc/v1/MarkerGrpcClient.kt @@ -2,22 +2,26 @@ package io.provenance.explorer.grpc.v1 import io.grpc.ManagedChannelBuilder import io.provenance.explorer.config.GrpcLoggingInterceptor -import io.provenance.explorer.grpc.extensions.getPaginationBuilder +import io.provenance.explorer.grpc.extensions.getPagination +import io.provenance.explorer.grpc.extensions.getPaginationNoCount import io.provenance.explorer.grpc.extensions.toMarker import io.provenance.marker.v1.MarkerAccount -import io.provenance.marker.v1.QueryGrpc -import io.provenance.marker.v1.QueryHoldingRequest +import io.provenance.marker.v1.QueryGrpcKt import io.provenance.marker.v1.QueryHoldingResponse -import io.provenance.marker.v1.QueryMarkerRequest +import io.provenance.marker.v1.queryHoldingRequest +import io.provenance.marker.v1.queryMarkerRequest +import kotlinx.coroutines.sync.Semaphore +import kotlinx.coroutines.sync.withPermit import org.springframework.stereotype.Component import java.net.URI import java.util.concurrent.TimeUnit import io.provenance.marker.v1.QueryParamsRequest as MarkerRequest @Component -class MarkerGrpcClient(channelUri: URI) { +class MarkerGrpcClient(channelUri: URI, private val semaphore: Semaphore) { - private val markerClient: QueryGrpc.QueryBlockingStub + private val markerClient: QueryGrpcKt.QueryCoroutineStub + private val markerClientFuture: QueryGrpcKt.QueryCoroutineStub init { val channel = @@ -29,44 +33,44 @@ class MarkerGrpcClient(channelUri: URI) { it.usePlaintext() } } - .idleTimeout(60, TimeUnit.SECONDS) - .keepAliveTime(10, TimeUnit.SECONDS) - .keepAliveTimeout(10, TimeUnit.SECONDS) + .idleTimeout(5, TimeUnit.MINUTES) + .keepAliveTime(60, TimeUnit.SECONDS) + .keepAliveTimeout(20, TimeUnit.SECONDS) .intercept(GrpcLoggingInterceptor()) .build() - markerClient = QueryGrpc.newBlockingStub(channel) + markerClient = QueryGrpcKt.QueryCoroutineStub(channel) + markerClientFuture = QueryGrpcKt.QueryCoroutineStub(channel) } - fun getMarkerDetail(id: String): MarkerAccount? = - try { - markerClient.marker(QueryMarkerRequest.newBuilder().setId(id).build()).marker.toMarker() - } catch (e: Exception) { - null + suspend fun getMarkerDetail(id: String): MarkerAccount? = + semaphore.withPermit { + try { + markerClient.marker(queryMarkerRequest { this.id = id }).marker.toMarker() + } catch (e: Exception) { + null + } } - fun getMarkerHolders(denom: String, offset: Int, count: Int): QueryHoldingResponse = - try { + suspend fun getMarkerHolders(denom: String, offset: Int, count: Int): QueryHoldingResponse = + semaphore.withPermit { markerClient.holding( - QueryHoldingRequest.newBuilder() - .setId(denom) - .setPagination(getPaginationBuilder(offset, count)) - .build() + queryHoldingRequest { + this.id = denom + this.pagination = getPaginationNoCount(offset, count) + } ) - } catch (e: Exception) { - QueryHoldingResponse.getDefaultInstance() } - fun getAllMarkerHolders(denom: String): QueryHoldingResponse = - try { + suspend fun getMarkerHoldersCount(denom: String): QueryHoldingResponse = + semaphore.withPermit { markerClient.holding( - QueryHoldingRequest.newBuilder() - .setId(denom) - .build() + queryHoldingRequest { + this.id = denom + this.pagination = getPagination(0, 1) + } ) - } catch (e: Exception) { - QueryHoldingResponse.getDefaultInstance() } - fun getMarkerParams() = markerClient.params(MarkerRequest.newBuilder().build()) + suspend fun getMarkerParams() = markerClient.params(MarkerRequest.newBuilder().build()) } diff --git a/service/src/main/kotlin/io/provenance/explorer/grpc/v1/MetadataGrpcClient.kt b/service/src/main/kotlin/io/provenance/explorer/grpc/v1/MetadataGrpcClient.kt index 8c8a5d1b..917d7c8a 100644 --- a/service/src/main/kotlin/io/provenance/explorer/grpc/v1/MetadataGrpcClient.kt +++ b/service/src/main/kotlin/io/provenance/explorer/grpc/v1/MetadataGrpcClient.kt @@ -2,22 +2,25 @@ package io.provenance.explorer.grpc.v1 import io.grpc.ManagedChannelBuilder import io.provenance.explorer.config.GrpcLoggingInterceptor -import io.provenance.explorer.grpc.extensions.getPaginationBuilder -import io.provenance.metadata.v1.OwnershipRequest -import io.provenance.metadata.v1.QueryGrpc -import io.provenance.metadata.v1.RecordSpecificationsForContractSpecificationRequest -import io.provenance.metadata.v1.ScopeRequest -import io.provenance.metadata.v1.ScopeSpecificationRequest -import io.provenance.metadata.v1.ScopesAllRequest +import io.provenance.explorer.grpc.extensions.getPagination +import io.provenance.metadata.v1.QueryGrpcKt.QueryCoroutineStub +import io.provenance.metadata.v1.contractSpecificationRequest +import io.provenance.metadata.v1.ownershipRequest +import io.provenance.metadata.v1.queryParamsRequest +import io.provenance.metadata.v1.recordSpecificationRequest +import io.provenance.metadata.v1.recordSpecificationsForContractSpecificationRequest +import io.provenance.metadata.v1.scopeRequest +import io.provenance.metadata.v1.scopeSpecificationRequest +import kotlinx.coroutines.sync.Semaphore +import kotlinx.coroutines.sync.withPermit import org.springframework.stereotype.Component import java.net.URI import java.util.concurrent.TimeUnit -import io.provenance.metadata.v1.QueryParamsRequest as MetadataRequest @Component -class MetadataGrpcClient(channelUri: URI) { +class MetadataGrpcClient(channelUri: URI, private val semaphore: Semaphore) { - private val metadataClient: QueryGrpc.QueryBlockingStub + private val metadataClient: QueryCoroutineStub init { val channel = @@ -35,46 +38,48 @@ class MetadataGrpcClient(channelUri: URI) { .intercept(GrpcLoggingInterceptor()) .build() - metadataClient = QueryGrpc.newBlockingStub(channel) + metadataClient = QueryCoroutineStub(channel) } - fun getAllScopes(offset: Int = 0, limit: Int = 10) = - metadataClient.scopesAll( - ScopesAllRequest.newBuilder() - .setPagination(getPaginationBuilder(offset, limit)) - .build() - ) - - fun getScopesByOwner(address: String, offset: Int = 0, limit: Int = 10) = - metadataClient.ownership( - OwnershipRequest.newBuilder() - .setAddress(address) - .setPagination(getPaginationBuilder(offset, limit)) - .build() - ) + suspend fun getScopesByOwner(address: String, offset: Int = 0, limit: Int = 10) = + semaphore.withPermit { + metadataClient.ownership( + ownershipRequest { + this.address = address + this.pagination = getPagination(offset, limit) + } + ) + } - fun getScopeById(uuid: String, includeSessions: Boolean = false) = + suspend fun getScopeById(uuid: String, includeRecords: Boolean = false, includeSessions: Boolean = false) = metadataClient.scope( - ScopeRequest.newBuilder() - .setScopeId(uuid) - .setIncludeRecords(true) - .setIncludeSessions(includeSessions) - .build() + scopeRequest { + this.scopeId = uuid + this.includeRecords = includeRecords + this.includeSessions = includeSessions + } ) - fun getScopeSpecById(addr: String) = - metadataClient.scopeSpecification( - ScopeSpecificationRequest.newBuilder() - .setSpecificationId(addr) - .build() - ) + suspend fun getScopeSpecById(addr: String) = + metadataClient.scopeSpecification(scopeSpecificationRequest { this.specificationId = addr }) - fun getRecordSpecsForContractSpec(contractSpec: String) = - metadataClient.recordSpecificationsForContractSpecification( - RecordSpecificationsForContractSpecificationRequest.newBuilder() - .setSpecificationId(contractSpec) - .build() + suspend fun getContractSpecById(addr: String, includeRecords: Boolean = false) = + metadataClient.contractSpecification( + contractSpecificationRequest { + this.specificationId = addr + this.includeRecordSpecs = includeRecords + } ) - fun getMetadataParams() = metadataClient.params(MetadataRequest.newBuilder().build()) + suspend fun getRecordSpecById(addr: String) = + metadataClient.recordSpecification(recordSpecificationRequest { this.specificationId = addr }) + + suspend fun getRecordSpecsForContractSpec(contractSpec: String) = + semaphore.withPermit { + metadataClient.recordSpecificationsForContractSpecification( + recordSpecificationsForContractSpecificationRequest { this.specificationId = contractSpec } + ) + } + + suspend fun getMetadataParams() = metadataClient.params(queryParamsRequest { }) } diff --git a/service/src/main/kotlin/io/provenance/explorer/grpc/v1/TransactionGrpcClient.kt b/service/src/main/kotlin/io/provenance/explorer/grpc/v1/TransactionGrpcClient.kt index babd5b44..ea5b82e1 100644 --- a/service/src/main/kotlin/io/provenance/explorer/grpc/v1/TransactionGrpcClient.kt +++ b/service/src/main/kotlin/io/provenance/explorer/grpc/v1/TransactionGrpcClient.kt @@ -53,10 +53,10 @@ class TransactionGrpcClient(channelUri: URI) { val txResps = results.txResponsesList.toMutableList() - if (txResps.count() == 0) + if (txResps.isEmpty()) throw TendermintApiException( "Blockchain failed to retrieve txs for height $height. Expected $total, " + - "Returned 0. This is not normal." + "Returned 0. This happens sometimes. The block will retry." ) while (txResps.count() < total) { diff --git a/service/src/main/kotlin/io/provenance/explorer/service/AccountService.kt b/service/src/main/kotlin/io/provenance/explorer/service/AccountService.kt index c5fb41e1..73144d41 100644 --- a/service/src/main/kotlin/io/provenance/explorer/service/AccountService.kt +++ b/service/src/main/kotlin/io/provenance/explorer/service/AccountService.kt @@ -28,6 +28,8 @@ import io.provenance.explorer.grpc.extensions.getModuleAccName import io.provenance.explorer.grpc.v1.AccountGrpcClient import io.provenance.explorer.grpc.v1.AttributeGrpcClient import io.provenance.explorer.grpc.v1.MetadataGrpcClient +import kotlinx.coroutines.async +import kotlinx.coroutines.runBlocking import org.jetbrains.exposed.sql.transactions.transaction import org.springframework.stereotype.Service @@ -43,51 +45,63 @@ class AccountService( fun getAccountRaw(address: String) = transaction { AccountRecord.findByAddress(address) } ?: saveAccount(address) - fun saveAccount(address: String, isContract: Boolean = false) = + fun saveAccount(address: String, isContract: Boolean = false) = runBlocking { accountClient.getAccountInfo(address)?.let { AccountRecord.insertIgnore(it, isContract) } - ?: if (address.isAddressAsType(props.provAccPrefix())) AccountRecord.insertUnknownAccount(address, isContract) + ?: if (address.isAddressAsType(props.provAccPrefix())) AccountRecord.insertUnknownAccount( + address, + isContract + ) else throw ResourceNotFoundException("Invalid account: '$address'") + } - fun getAccountDetail(address: String) = getAccountRaw(address).let { - AccountDetail( - it.type, - it.accountAddress, - it.accountNumber, - it.baseAccount?.sequence?.toInt(), - AccountRecord.findSigsByAddress(it.accountAddress).toSigObj(props.provAccPrefix()), - it.data?.getModuleAccName(), - attrClient.getAllAttributesForAddress(it.accountAddress).map { attr -> attr.toResponse() }, - TokenCounts( - getBalances(it.accountAddress, 0, 1).pagination.total, - metadataClient.getScopesByOwner(it.accountAddress).pagination.total.toInt() + fun getAccountDetail(address: String) = runBlocking { + getAccountRaw(address).let { + val attributes = async { attrClient.getAllAttributesForAddress(it.accountAddress) } + val balances = async { getBalances(it.accountAddress, 0, 1) } + AccountDetail( + it.type, + it.accountAddress, + it.accountNumber, + it.baseAccount?.sequence?.toInt(), + AccountRecord.findSigsByAddress(it.accountAddress).toSigObj(props.provAccPrefix()), + it.data?.getModuleAccName(), + attributes.await().map { attr -> attr.toResponse() }, + TokenCounts( + balances.await().pagination.total, + metadataClient.getScopesByOwner(it.accountAddress).pagination.total.toInt() + ), + it.isContract ) - ) + } } - fun getNamesOwnedByAccount(address: String, page: Int, limit: Int) = + fun getNamesOwnedByAccount(address: String, page: Int, limit: Int) = runBlocking { attrClient.getNamesForAddress(address, page.toOffset(limit), limit).let { res -> val names = res.nameList.toList() PagedResults(res.pagination.total.pageCountOfResults(limit), names, res.pagination.total) } + } - fun getBalances(address: String, page: Int, limit: Int) = + suspend fun getBalances(address: String, page: Int, limit: Int) = accountClient.getAccountBalances(address, page.toOffset(limit), limit) - fun getAccountBalances(address: String, page: Int, limit: Int) = + fun getAccountBalances(address: String, page: Int, limit: Int) = runBlocking { getBalances(address, page, limit).let { res -> val bals = res.balancesList.map { it.toData() } PagedResults(res.pagination.total.pageCountOfResults(limit), bals, res.pagination.total) } + } - fun getCurrentSupply(denom: String) = accountClient.getCurrentSupply(denom).amount + fun getCurrentSupply(denom: String) = runBlocking { accountClient.getCurrentSupply(denom).amount } - fun getDenomMetadataSingle(denom: String) = accountClient.getDenomMetadata(denom).metadata + fun getDenomMetadataSingle(denom: String) = runBlocking { accountClient.getDenomMetadata(denom).metadata } - fun getDenomMetadata(denom: String?) = + fun getDenomMetadata(denom: String?) = runBlocking { if (denom != null) listOf(accountClient.getDenomMetadata(denom).metadata) else accountClient.getAllDenomMetadata().metadatasList + } - fun getDelegations(address: String, page: Int, limit: Int) = + fun getDelegations(address: String, page: Int, limit: Int) = runBlocking { accountClient.getDelegations(address, page.toOffset(limit), limit).let { res -> val list = res.delegationResponsesList.map { Delegation( @@ -103,8 +117,9 @@ class AccountService( } PagedResults(res.pagination.total.pageCountOfResults(limit), list, res.pagination.total) } + } - fun getUnbondingDelegations(address: String) = + fun getUnbondingDelegations(address: String) = runBlocking { accountClient.getUnbondingDelegations(address, 0, 100).let { res -> res.unbondingResponsesList.flatMap { list -> list.entriesList.map { @@ -121,8 +136,9 @@ class AccountService( } } } + } - fun getRedelegations(address: String) = + fun getRedelegations(address: String) = runBlocking { accountClient.getRedelegations(address, 0, 100).let { res -> res.redelegationResponsesList.flatMap { list -> list.entriesList.map { @@ -139,17 +155,20 @@ class AccountService( } } } + } - fun getRewards(address: String) = accountClient.getRewards(address).let { res -> - AccountRewards( - res.rewardsList.map { list -> - Reward( - list.validatorAddress, - list.rewardList.map { r -> CoinStr(r.amount.toDecCoin(), r.denom) } - ) - }, - res.totalList.map { t -> CoinStr(t.amount.toDecCoin(), t.denom) } - ) + fun getRewards(address: String) = runBlocking { + accountClient.getRewards(address).let { res -> + AccountRewards( + res.rewardsList.map { list -> + Reward( + list.validatorAddress, + list.rewardList.map { r -> CoinStr(r.amount.toDecCoin(), r.denom) } + ) + }, + res.totalList.map { t -> CoinStr(t.amount.toDecCoin(), t.denom) } + ) + } } } diff --git a/service/src/main/kotlin/io/provenance/explorer/service/AssetService.kt b/service/src/main/kotlin/io/provenance/explorer/service/AssetService.kt index a3985812..b68a8f9b 100644 --- a/service/src/main/kotlin/io/provenance/explorer/service/AssetService.kt +++ b/service/src/main/kotlin/io/provenance/explorer/service/AssetService.kt @@ -29,6 +29,11 @@ import io.provenance.explorer.grpc.v1.AttributeGrpcClient import io.provenance.explorer.grpc.v1.MarkerGrpcClient import io.provenance.explorer.grpc.v1.MetadataGrpcClient import io.provenance.marker.v1.MarkerStatus +import kotlinx.coroutines.async +import kotlinx.coroutines.flow.asFlow +import kotlinx.coroutines.flow.map +import kotlinx.coroutines.flow.toList +import kotlinx.coroutines.runBlocking import org.jetbrains.exposed.sql.transactions.transaction import org.springframework.stereotype.Service import java.math.BigDecimal @@ -75,7 +80,7 @@ class AssetService( MarkerCacheRecord.findByDenom(denom)?.let { Pair(it.id, it) } } - private fun getAndInsertMarker(denom: String) = + private fun getAndInsertMarker(denom: String) = runBlocking { markerClient.getMarkerDetail(denom)?.let { MarkerCacheRecord.insertIgnore( it.baseAccount.address, @@ -97,11 +102,14 @@ class AssetService( TxMarkerJoinRecord.findLatestTxByDenom(denom) ) } + } fun getAssetDetail(denom: String) = - getAssetFromDB(denom) - ?.let { (id, record) -> + runBlocking { + getAssetFromDB(denom)?.let { (id, record) -> val txCount = TxMarkerJoinRecord.findCountByDenom(id.value) + val attributes = async { attrClient.getAllAttributesForAddress(record.markerAddress) } + val balances = async { accountService.getBalances(record.markerAddress!!, 1, 1) } AssetDetail( record.denom, record.markerAddress, @@ -111,35 +119,30 @@ class AssetService( ) else null, CoinStr(record.supply.toBigInteger().toString(), record.denom), record.data?.isMintable() ?: false, - if (record.markerAddress != null) markerClient.getMarkerHolders( - denom, - 0, - 10 - ).pagination.total.toInt() else 0, + if (record.markerAddress != null) markerClient.getMarkerHoldersCount(denom).pagination.total.toInt() else 0, txCount, - attrClient.getAllAttributesForAddress(record.markerAddress).map { attr -> attr.toResponse() }, + attributes.await().map { attr -> attr.toResponse() }, accountService.getDenomMetadataSingle(denom).toObjectNode(protoPrinter), TokenCounts( - if (record.markerAddress != null) accountService.getBalances( - record.markerAddress!!, - 0, - 1 - ).pagination.total else 0, + if (record.markerAddress != null) balances.await().pagination.total else 0, if (record.markerAddress != null) metadataClient.getScopesByOwner(record.markerAddress!!).pagination.total.toInt() else 0 ), record.status.prettyStatus(), record.markerType.prettyMarkerType() ) } ?: throw ResourceNotFoundException("Invalid asset: $denom") + } - fun getAssetHolders(denom: String, page: Int, count: Int) = accountService.getCurrentSupply(denom).let { supply -> - val res = markerClient.getMarkerHolders(denom, page.toOffset(count), count) - val list = res.balancesList.map { bal -> - val balance = bal.coinsList.first { coin -> coin.denom == denom }.amount - AssetHolder(bal.address, CountStrTotal(balance, supply, denom)) - }.sortedByDescending { it.balance.count.toBigDecimal() } - PagedResults(res.pagination.total.pageCountOfResults(count), list, res.pagination.total) - } + fun getAssetHolders(denom: String, page: Int, count: Int) = + runBlocking { + val supply = accountService.getCurrentSupply(denom) + val res = markerClient.getMarkerHolders(denom, page.toOffset(count), count) + val list = res.balancesList.asFlow().map { bal -> + val balance = bal.coinsList.first { coin -> coin.denom == denom }.amount + AssetHolder(bal.address, CountStrTotal(balance, supply, denom)) + }.toList().sortedWith(compareBy { it.balance.count.toBigDecimal() }).asReversed() + PagedResults(res.pagination.total.pageCountOfResults(count), list, res.pagination.total) + } fun getTokenDistributionStats() = transaction { TokenDistributionAmountsRecord.getStats() } @@ -216,18 +219,20 @@ class AssetService( fun getMetadata(denom: String?) = accountService.getDenomMetadata(denom).map { it.toObjectNode(protoPrinter) } // Updates the Marker cache - fun updateAssets(denoms: Set, txTime: Timestamp) = transaction { - logger.info("saving assets") - denoms.forEach { marker -> - val data = markerClient.getMarkerDetail(marker) - MarkerCacheRecord.findByDenom(marker)?.apply { - if (data != null) this.status = data.status.toString() - this.supply = accountService.getCurrentSupply(marker).toBigDecimal() - this.lastTx = txTime.toDateTime() - this.data = data + fun updateAssets(denoms: Set, txTime: Timestamp) = + transaction { + denoms.forEach { marker -> + runBlocking { + val data = markerClient.getMarkerDetail(marker) + MarkerCacheRecord.findByDenom(marker)?.apply { + if (data != null) this.status = data.status.toString() + this.supply = accountService.getCurrentSupply(marker).toBigDecimal() + this.lastTx = txTime.toDateTime() + this.data = data + } + } } } - } } fun BigDecimal.asPercentOf(divisor: BigDecimal): BigDecimal = this.divide(divisor, 20, RoundingMode.CEILING) diff --git a/service/src/main/kotlin/io/provenance/explorer/service/ExplorerService.kt b/service/src/main/kotlin/io/provenance/explorer/service/ExplorerService.kt index 5f746bb6..41b60fe6 100644 --- a/service/src/main/kotlin/io/provenance/explorer/service/ExplorerService.kt +++ b/service/src/main/kotlin/io/provenance/explorer/service/ExplorerService.kt @@ -55,6 +55,7 @@ import io.provenance.explorer.grpc.v1.MetadataGrpcClient import io.provenance.explorer.grpc.v1.ValidatorGrpcClient import io.provenance.explorer.service.async.AsyncCaching import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.async import kotlinx.coroutines.runBlocking import org.jetbrains.exposed.sql.transactions.transaction import org.joda.time.DateTime @@ -174,14 +175,14 @@ class ExplorerService( fun getChainId() = asyncCaching.getChainIdString() - fun getParams(): Params { - val authParams = accountClient.getAuthParams().params - val bankParams = accountClient.getBankParams().params + fun getParams(): Params = runBlocking { + val authParams = async { accountClient.getAuthParams().params }.await() + val bankParams = async { accountClient.getBankParams().params }.await() val distParams = validatorClient.getDistParams().params val votingParams = govClient.getParams(GovParamType.voting).votingParams val tallyParams = govClient.getParams(GovParamType.tallying).tallyParams val depositParams = govClient.getParams(GovParamType.deposit).depositParams - val mintParams = accountClient.getMintParams().params + val mintParams = async { accountClient.getMintParams().params }.await() val slashingParams = validatorClient.getSlashingParams().params val stakingParams = validatorClient.getStakingParams().params val transferParams = ibcClient.getTransferParams().params @@ -191,7 +192,7 @@ class ExplorerService( val metadataParams = metadataClient.getMetadataParams().params val nameParams = attrClient.getNameParams().params - return Params( + Params( CosmosParams( AuthParams( authParams.maxMemoCharacters, diff --git a/service/src/main/kotlin/io/provenance/explorer/service/NftService.kt b/service/src/main/kotlin/io/provenance/explorer/service/NftService.kt index 4be321cb..0de68979 100644 --- a/service/src/main/kotlin/io/provenance/explorer/service/NftService.kt +++ b/service/src/main/kotlin/io/provenance/explorer/service/NftService.kt @@ -1,5 +1,6 @@ package io.provenance.explorer.service +import com.google.protobuf.util.JsonFormat import io.provenance.explorer.domain.core.MdParent import io.provenance.explorer.domain.core.MetadataAddress import io.provenance.explorer.domain.core.getParentForType @@ -14,6 +15,8 @@ import io.provenance.explorer.domain.entities.NftScopeSpecRecord import io.provenance.explorer.domain.entities.TxNftJoinRecord import io.provenance.explorer.domain.extensions.formattedString import io.provenance.explorer.domain.extensions.pageCountOfResults +import io.provenance.explorer.domain.extensions.protoTypesFieldsToCheckForMetadata +import io.provenance.explorer.domain.extensions.toObjectNodeMAddressValues import io.provenance.explorer.domain.extensions.toOffset import io.provenance.explorer.domain.models.explorer.PagedResults import io.provenance.explorer.domain.models.explorer.PartyAndRole @@ -23,39 +26,52 @@ import io.provenance.explorer.domain.models.explorer.RecordStatus import io.provenance.explorer.domain.models.explorer.ScopeDetail import io.provenance.explorer.domain.models.explorer.ScopeListview import io.provenance.explorer.domain.models.explorer.ScopeRecord +import io.provenance.explorer.domain.models.explorer.toDataObject import io.provenance.explorer.domain.models.explorer.toOwnerRoles import io.provenance.explorer.domain.models.explorer.toSpecDescrip import io.provenance.explorer.grpc.v1.MetadataGrpcClient +import kotlinx.coroutines.flow.asFlow +import kotlinx.coroutines.flow.flatMapMerge +import kotlinx.coroutines.flow.map +import kotlinx.coroutines.flow.toList +import kotlinx.coroutines.runBlocking import org.jetbrains.exposed.sql.transactions.transaction import org.springframework.stereotype.Service @Service class NftService( - private val metadataClient: MetadataGrpcClient + private val metadataClient: MetadataGrpcClient, + private val protoPrinter: JsonFormat.Printer ) { - fun getScopeDescrip(addr: String) = + suspend fun getScopeDescrip(addr: String) = metadataClient.getScopeSpecById(addr).scopeSpecification.specification.description - fun getScopesForOwningAddress(address: String, page: Int, count: Int) = + // TODO: need to update to pull from db tables for owner/VO/data access + fun getScopesForOwningAddress(address: String, page: Int, count: Int) = runBlocking { metadataClient.getScopesByOwner(address, page.toOffset(count), count).let { val records = it.scopeUuidsList.map { uuid -> scopeToListview(uuid) } PagedResults(it.pagination.total.pageCountOfResults(count), records, it.pagination.total) } + } - private fun scopeToListview(addr: String) = + private fun scopeToListview(addr: String) = runBlocking { metadataClient.getScopeById(addr).let { val lastTx = TxNftJoinRecord.findTxByUuid(it.scope.scopeIdInfo.scopeUuid, 0, 1).firstOrNull() ScopeListview( it.scope.scopeIdInfo.scopeAddr, getScopeDescrip(it.scope.scopeSpecIdInfo.scopeSpecAddr)?.name, it.scope.scopeSpecIdInfo.scopeSpecAddr, - lastTx?.txTimestamp?.toString() ?: "" + lastTx?.txTimestamp?.toString() ?: "", + it.scope.scope.ownersList.map { own -> own.address }.contains(addr), + it.scope.scope.dataAccessList.contains(addr), + it.scope.scope.valueOwnerAddress == addr ) } + } - fun getScopeDetail(addr: String) = metadataClient.getScopeById(addr) - .let { + fun getScopeDetail(addr: String) = runBlocking { + metadataClient.getScopeById(addr).let { val spec = getScopeDescrip(it.scope.scopeSpecIdInfo.scopeSpecAddr) ScopeDetail( it.scope.scopeIdInfo.scopeAddr, @@ -63,26 +79,29 @@ class NftService( it.scope.scopeSpecIdInfo.scopeSpecAddr, spec?.toSpecDescrip(), it.scope.scope.ownersList.toOwnerRoles(), + it.scope.scope.dataAccessList, it.scope.scope.valueOwnerAddress ) } + } - fun getRecordsForScope(addr: String): List { + fun getRecordsForScope(addr: String): List = runBlocking { // get scope - val scope = metadataClient.getScopeById(addr, true) + val scope = metadataClient.getScopeById(addr, true, true) // get scope spec -> contract specs val scopeSpec = metadataClient.getScopeSpecById(scope.scope.scopeSpecIdInfo.scopeSpecAddr) val contractSpecs = scopeSpec.scopeSpecification.specification.contractSpecIdsList.map { it.toMAddress() } // get record specs for each contract spec - val recordSpecs = contractSpecs - .flatMap { cs -> metadataClient.getRecordSpecsForContractSpec(cs.toString()).recordSpecificationsList } + val recordSpecs = contractSpecs.asFlow() + .flatMapMerge { cs -> metadataClient.getRecordSpecsForContractSpec(cs.toString()).recordSpecificationsList.asFlow() } + .toList() .groupBy { it.specification.name } // get records/sessions val records = scope.recordsList.associateBy { it.record.name } val sessions = scope.sessionsList.associateBy { it.sessionIdInfo.sessionAddr } // Given current record specs, match existing records by name / record spec addr. - val list = recordSpecs.map { (k, v) -> + val list = recordSpecs.toList().asFlow().map { (k, v) -> val record = records[k] // if the record for this spec name exists, make a record detail @@ -92,7 +111,8 @@ class NftService( r.recordIdInfo.recordAddr, r.recordSpecIdInfo.recordSpecAddr, session.session.audit.createdDate.formattedString(), - session.session.partiesList.map { p -> PartyAndRole(p.address, p.role.name) } + session.session.partiesList.map { p -> PartyAndRole(p.address, p.role.name) }, + r.record.outputsList.map { it.toDataObject(k) } ) } @@ -114,7 +134,8 @@ class NftService( ScopeRecord(RecordStatus.FILLED, k, null, recDetail) else -> ScopeRecord(RecordStatus.NON_CONFORMING, k, null, recDetail) } - } + }.toList() + // For any remaining records that do not match a spec name, set the record, status ORPHAN val seen = list.map { it.recordName } val noSpecOrphans = records.filter { !seen.contains(it.key) }.map { (k, v) -> @@ -123,13 +144,29 @@ class NftService( v.recordIdInfo.recordAddr, v.recordSpecIdInfo.recordSpecAddr, session.session.audit.createdDate.formattedString(), - session.session.partiesList.map { p -> PartyAndRole(p.address, p.role.name) } + session.session.partiesList.map { p -> PartyAndRole(p.address, p.role.name) }, + v.record.outputsList.map { it.toDataObject(k) } ) ScopeRecord(RecordStatus.ORPHAN, k, null, recDetail) } // Sort by name - return (list + noSpecOrphans).sortedBy { it.recordName } + (list + noSpecOrphans).sortedBy { it.recordName } + } + + fun getScopeSpecJson(scopeSpec: String) = runBlocking { + metadataClient.getScopeSpecById(scopeSpec) + .toObjectNodeMAddressValues(protoPrinter, protoTypesFieldsToCheckForMetadata) + } + + fun getContractSpecJson(contractSpec: String) = runBlocking { + metadataClient.getContractSpecById(contractSpec, true) + .toObjectNodeMAddressValues(protoPrinter, protoTypesFieldsToCheckForMetadata) + } + + fun getRecordSpecJson(recordSpec: String) = runBlocking { + metadataClient.getRecordSpecById(recordSpec) + .toObjectNodeMAddressValues(protoPrinter, protoTypesFieldsToCheckForMetadata) } fun translateAddress(addr: String) = MetadataAddress.fromBech32(addr) @@ -148,7 +185,7 @@ class NftService( NftContractSpecRecord .getOrInsert(md.getPrimaryUuid().toString(), md.getPrimaryUuid().toMAddressContractSpec().toString()) .let { Triple(MdParent.CONTRACT_SPEC, it.id.value, it.uuid) } - else -> null.also { logger().debug("This prefix doesnt have a parent type: ${md.getPrefix()}") } + else -> null.also { logger().debug("This prefix doesn't have a parent type: ${md.getPrefix()}") } } } @@ -163,7 +200,7 @@ class NftService( MdParent.CONTRACT_SPEC -> NftContractSpecRecord .markDeleted(md.getPrimaryUuid().toString(), md.getPrimaryUuid().toMAddressContractSpec().toString()) - else -> null.also { logger().debug("This prefix doesnt have a parent type: ${md.getPrefix()}") } + else -> null.also { logger().debug("This prefix doesn't have a parent type: ${md.getPrefix()}") } } } @@ -172,7 +209,7 @@ class NftService( MdParent.SCOPE -> NftScopeRecord.findByUuid(md.getPrimaryUuid().toString())!!.id.value MdParent.SCOPE_SPEC -> NftScopeSpecRecord.findByUuid(md.getPrimaryUuid().toString())!!.id.value MdParent.CONTRACT_SPEC -> NftContractSpecRecord.findByUuid(md.getPrimaryUuid().toString())!!.id.value - else -> null.also { logger().debug("This prefix doesnt have a parent type: ${md?.getPrefix()}") } + else -> null.also { logger().debug("This prefix doesn't have a parent type: ${md?.getPrefix()}") } } } } diff --git a/service/src/main/kotlin/io/provenance/explorer/service/SmartContractService.kt b/service/src/main/kotlin/io/provenance/explorer/service/SmartContractService.kt index c17a9ca5..5eab1516 100644 --- a/service/src/main/kotlin/io/provenance/explorer/service/SmartContractService.kt +++ b/service/src/main/kotlin/io/provenance/explorer/service/SmartContractService.kt @@ -12,6 +12,8 @@ import io.provenance.explorer.domain.entities.SmContractTable import io.provenance.explorer.domain.extensions.pageCountOfResults import io.provenance.explorer.domain.extensions.toObjectNodeNonTxMsg import io.provenance.explorer.domain.extensions.toOffset +import io.provenance.explorer.domain.models.explorer.Code +import io.provenance.explorer.domain.models.explorer.Contract import io.provenance.explorer.domain.models.explorer.PagedResults import io.provenance.explorer.domain.models.explorer.TxData import io.provenance.explorer.grpc.extensions.toMsgClearAdmin @@ -51,25 +53,38 @@ class SmartContractService( } fun getAllScContractsPaginated(page: Int, count: Int) = transaction { - SmContractRecord.getPaginated(count, page.toOffset(count)).let { - val total = SmContractRecord.count() - PagedResults(total.pageCountOfResults(count), it, total) - } + SmContractRecord.getPaginated(page.toOffset(count), count) + .map { it.toContractObject() } + .let { + val total = SmContractRecord.count() + PagedResults(total.pageCountOfResults(count), it, total) + } + } + + fun getAllScCodesPaginated(page: Int, count: Int) = transaction { + SmCodeRecord.getPaginated(page.toOffset(count), count) + .map { it.toCodeObject() } + .let { + val total = SmCodeRecord.count() + PagedResults(total.pageCountOfResults(count), it, total) + } } fun getCode(code: Int) = transaction { - SmCodeRecord.findById(code) ?: throw ResourceNotFoundException("Invalid code ID: $code") + SmCodeRecord.findById(code)?.toCodeObject() ?: throw ResourceNotFoundException("Invalid code ID: $code") } fun getContractsByCode(codeId: Int, page: Int, count: Int) = transaction { - SmContractRecord.getPaginated(count, page.toOffset(count), codeId).let { - val total = SmContractRecord.count(Op.build { SmContractTable.codeId eq codeId }) - PagedResults(total.pageCountOfResults(count), it, total) - } + SmContractRecord.getPaginated(page.toOffset(count), count, codeId) + .map { it.toContractObject() } + .let { + val total = SmContractRecord.count(Op.build { SmContractTable.codeId eq codeId }) + PagedResults(total.pageCountOfResults(count), it, total) + } } fun getContract(contract: String) = transaction { - SmContractRecord.findByContractAddress(contract) + SmContractRecord.findByContractAddress(contract)?.toContractObject() ?: throw ResourceNotFoundException("Invalid contract address: $contract") } @@ -78,6 +93,11 @@ class SmartContractService( } } +fun SmCodeRecord.toCodeObject() = Code(this.id.value, this.creationHeight, this.creator, this.dataHash) + +fun SmContractRecord.toContractObject() = + Contract(this.contractAddress, this.creationHeight, this.codeId, this.creator, this.admin, this.label) + fun ByteArray.isGZIPStream(): Boolean { return ( this[0] == GZIPInputStream.GZIP_MAGIC.toByte() && diff --git a/service/src/main/kotlin/io/provenance/explorer/service/TransactionService.kt b/service/src/main/kotlin/io/provenance/explorer/service/TransactionService.kt index 36d7d719..6ac5f11c 100644 --- a/service/src/main/kotlin/io/provenance/explorer/service/TransactionService.kt +++ b/service/src/main/kotlin/io/provenance/explorer/service/TransactionService.kt @@ -5,8 +5,10 @@ import cosmos.tx.v1beta1.ServiceOuterClass import io.provenance.explorer.config.ExplorerProperties import io.provenance.explorer.config.ResourceNotFoundException import io.provenance.explorer.domain.core.getParentForType +import io.provenance.explorer.domain.core.isMAddress import io.provenance.explorer.domain.core.logger import io.provenance.explorer.domain.core.toMAddress +import io.provenance.explorer.domain.core.toMAddressScope import io.provenance.explorer.domain.entities.AccountRecord import io.provenance.explorer.domain.entities.BlockCacheHourlyTxCountsRecord import io.provenance.explorer.domain.entities.MarkerCacheRecord @@ -95,8 +97,8 @@ class TransactionService( val msgTypeIds = transaction { TxMessageTypeRecord.findByType(msgTypes).map { it.id.value } }.toList() val addr = transaction { address?.getAddressType(props) } val markerId = if (denom != null) MarkerCacheRecord.findByDenom(denom)?.id?.value else null - val nft = nftAddr?.toMAddress() - ?.let { Triple(it.getParentForType()?.name, nftService.getNftDbId(it), it.getPrimaryUuid().toString()) } + val nftMAddress = if (nftAddr != null && nftAddr.isMAddress()) nftAddr.toMAddress() else nftAddr?.toMAddressScope() + val nft = nftMAddress?.let { Triple(it.getParentForType()?.name, nftService.getNftDbId(it), it.getPrimaryUuid().toString()) } val params = TxQueryParams( diff --git a/service/src/main/kotlin/io/provenance/explorer/service/ValidatorService.kt b/service/src/main/kotlin/io/provenance/explorer/service/ValidatorService.kt index e9c46860..5a5da6c8 100644 --- a/service/src/main/kotlin/io/provenance/explorer/service/ValidatorService.kt +++ b/service/src/main/kotlin/io/provenance/explorer/service/ValidatorService.kt @@ -8,6 +8,7 @@ import io.provenance.explorer.config.ResourceNotFoundException import io.provenance.explorer.domain.core.logger import io.provenance.explorer.domain.entities.BlockProposerRecord import io.provenance.explorer.domain.entities.MissedBlocksRecord +import io.provenance.explorer.domain.entities.SpotlightCacheRecord import io.provenance.explorer.domain.entities.StakingValidatorCacheRecord import io.provenance.explorer.domain.entities.ValidatorGasFeeCacheRecord import io.provenance.explorer.domain.entities.ValidatorStateRecord @@ -32,10 +33,16 @@ import io.provenance.explorer.domain.models.explorer.CountStrTotal import io.provenance.explorer.domain.models.explorer.CountTotal import io.provenance.explorer.domain.models.explorer.CurrentValidatorState import io.provenance.explorer.domain.models.explorer.Delegation +import io.provenance.explorer.domain.models.explorer.MissedBlockSet +import io.provenance.explorer.domain.models.explorer.MissedBlocksTimeframe import io.provenance.explorer.domain.models.explorer.PagedResults +import io.provenance.explorer.domain.models.explorer.Timeframe import io.provenance.explorer.domain.models.explorer.ValidatorCommission import io.provenance.explorer.domain.models.explorer.ValidatorDetails +import io.provenance.explorer.domain.models.explorer.ValidatorMissedBlocks +import io.provenance.explorer.domain.models.explorer.ValidatorMoniker import io.provenance.explorer.domain.models.explorer.ValidatorSummary +import io.provenance.explorer.domain.models.explorer.hourlyBlockCount import io.provenance.explorer.grpc.extensions.toAddress import io.provenance.explorer.grpc.v1.ValidatorGrpcClient import org.jetbrains.exposed.sql.transactions.transaction @@ -111,7 +118,7 @@ class ValidatorService( addr.operatorAddress, addr.accountAddr, grpcClient.getDelegatorWithdrawalAddress(addr.accountAddr), - stakingValidator.consensusPubkey.toAddress(props.provValConsPrefix()) ?: "", + addr.consensusAddr, CountTotal( (getMissedBlocks(addr.consensusAddr)?.totalCount ?: 0).toBigInteger(), currentHeight - (signingInfo?.startHeight?.toBigInteger() ?: BigInteger.ZERO) @@ -411,4 +418,62 @@ class ValidatorService( val data = res.associate { it.blockHeight to it.blockLatency!! } BlockLatencyData(address, data, average) } + + fun getDistinctValidatorsWithMissedBlocksInTimeframe(timeframe: Timeframe) = transaction { + val currentHeight = SpotlightCacheRecord.getSpotlight().latestBlock.height + val frame = when (timeframe) { + Timeframe.WEEK -> hourlyBlockCount * 24 * 7 + Timeframe.DAY -> hourlyBlockCount * 24 + Timeframe.HOUR -> hourlyBlockCount + Timeframe.FOREVER -> currentHeight - 1 + } + + val validators = MissedBlocksRecord + .findDistinctValidatorsWithMissedBlocksForPeriod(currentHeight - frame, currentHeight) + .let { ValidatorStateRecord.findByConsensusAddressIn(it.toList()) } + .map { ValidatorMissedBlocks(ValidatorMoniker(it.consensusAddr, it.operatorAddress, it.moniker)) } + + MissedBlocksTimeframe(currentHeight - frame, currentHeight, validators) + } + + fun getMissedBlocksForValidatorInTimeframe(timeframe: Timeframe, validatorAddr: String?) = transaction { + if (timeframe == Timeframe.FOREVER && validatorAddr == null) + throw IllegalArgumentException("If timeframe is FOREVER, you must have a validator operator address specified.") + if (validatorAddr != null && !validatorAddr.startsWith(props.provValOperPrefix())) + throw IllegalArgumentException("'validatorAddr' must begin with the validator operator address prefix : ${props.provValOperPrefix()}") + + val currentHeight = SpotlightCacheRecord.getSpotlight().latestBlock.height + val frame = when (timeframe) { + Timeframe.WEEK -> hourlyBlockCount * 24 * 7 + Timeframe.DAY -> hourlyBlockCount * 24 + Timeframe.HOUR -> hourlyBlockCount + Timeframe.FOREVER -> currentHeight - 1 + } + + val valConsAddr = if (validatorAddr != null) + StakingValidatorCacheRecord.findByOperAddr(validatorAddr)?.consensusAddress + else + null + + val results = MissedBlocksRecord + .findValidatorsWithMissedBlocksForPeriod(currentHeight - frame, currentHeight, valConsAddr) + val vals = ValidatorStateRecord.findByConsensusAddressIn(results.map { it.validator.valConsAddress }) + .associateBy { it.consensusAddr } + + val list = results + .groupBy( + { it.validator }, + { MissedBlockSet(it.blocks.minOrNull()!!, it.blocks.maxOrNull()!!, it.blocks.size) } + ) + .map { (k, v) -> ValidatorMissedBlocks(k, v) } + + list.forEach { res -> + vals[res.validator.valConsAddress].let { match -> + res.validator.operatorAddr = match?.operatorAddress + res.validator.moniker = match?.moniker + } + } + + MissedBlocksTimeframe(currentHeight - frame, currentHeight, list) + } } diff --git a/service/src/main/kotlin/io/provenance/explorer/service/async/AsyncCaching.kt b/service/src/main/kotlin/io/provenance/explorer/service/async/AsyncCaching.kt index 0a8d779c..a959d1dc 100644 --- a/service/src/main/kotlin/io/provenance/explorer/service/async/AsyncCaching.kt +++ b/service/src/main/kotlin/io/provenance/explorer/service/async/AsyncCaching.kt @@ -170,7 +170,7 @@ class AsyncCaching( .also { calculateBlockTxFee(it, blockHeight) } .map { addTxToCacheWithTimestamp(txClient.getTxByHash(it.txhash), blockTime) } } catch (e: Exception) { - logger.error("Failed to retrieve transactions at block: $blockHeight", e) + logger.error("Failed to retrieve transactions at block: $blockHeight", e.message) BlockTxRetryRecord.insert(blockHeight, e) listOf() } diff --git a/service/src/main/kotlin/io/provenance/explorer/service/utility/UtilityService.kt b/service/src/main/kotlin/io/provenance/explorer/service/utility/UtilityService.kt index da2e3072..4f2e6f06 100644 --- a/service/src/main/kotlin/io/provenance/explorer/service/utility/UtilityService.kt +++ b/service/src/main/kotlin/io/provenance/explorer/service/utility/UtilityService.kt @@ -26,6 +26,7 @@ import io.provenance.explorer.service.AssetService import io.provenance.explorer.service.GovService import io.provenance.explorer.service.async.AsyncCaching import io.provenance.explorer.service.gzipUncompress +import kotlinx.coroutines.runBlocking import net.pearx.kasechange.toSnakeCase import net.pearx.kasechange.universalWordSplitter import org.bouncycastle.util.encoders.Hex @@ -74,25 +75,29 @@ class UtilityService( } // searches for accounts that may or may not have the denom balance - fun searchAccountsForDenom(accounts: List, denom: String): List> { - var offset = 0 - val limit = 100 - - val results = markerClient.getMarkerHolders(denom, offset, limit) - val total = results.pagination?.total ?: results.balancesCount.toLong() - val holders = results.balancesList.toMutableList() - - while (holders.count() < total) { - offset += limit - markerClient.getMarkerHolders(denom, offset, limit).let { holders.addAll(it.balancesList) } - } + fun searchAccountsForDenom(accounts: List, denom: String): List> = + runBlocking { + var offset = 0 + val limit = 100 + + val results = markerClient.getMarkerHolders(denom, offset, limit) + val total = results.pagination?.total ?: results.balancesCount.toLong() + val holders = results.balancesList.toMutableList() + + while (holders.count() < total) { + offset += limit + markerClient.getMarkerHolders(denom, offset, limit).let { holders.addAll(it.balancesList) } + } - val map = holders.associateBy { it.address } + val map = holders.associateBy { it.address } - return accounts.toSet().map { a -> - mapOf("address" to a, denom to (map[a]?.coinsList?.firstOrNull { c -> c.denom == denom }?.amount ?: "Nothing")) + accounts.toSet().map { a -> + mapOf( + "address" to a, + denom to (map[a]?.coinsList?.firstOrNull { c -> c.denom == denom }?.amount ?: "Nothing") + ) + } } - } fun stringToJson(str: String) = str.toObjectNode() diff --git a/service/src/main/kotlin/io/provenance/explorer/web/v2/MigrationController.kt b/service/src/main/kotlin/io/provenance/explorer/web/v2/MigrationController.kt index d841cf3b..19dd1a11 100644 --- a/service/src/main/kotlin/io/provenance/explorer/web/v2/MigrationController.kt +++ b/service/src/main/kotlin/io/provenance/explorer/web/v2/MigrationController.kt @@ -15,7 +15,7 @@ import org.springframework.web.bind.annotation.RestController @Validated @RestController -@RequestMapping(path = ["/api/v2"], produces = [MediaType.APPLICATION_JSON_VALUE]) +@RequestMapping(path = ["/api/v2/migration"], produces = [MediaType.APPLICATION_JSON_VALUE]) @Api( value = "Migration controller", produces = "application/json", consumes = "application/json", tags = ["Migrations"], description = "This should NEVER be used by the UI" diff --git a/service/src/main/kotlin/io/provenance/explorer/web/v2/NftController.kt b/service/src/main/kotlin/io/provenance/explorer/web/v2/NftController.kt index 1ee4500c..1a01b05c 100644 --- a/service/src/main/kotlin/io/provenance/explorer/web/v2/NftController.kt +++ b/service/src/main/kotlin/io/provenance/explorer/web/v2/NftController.kt @@ -38,4 +38,16 @@ class NftController(private val nftService: NftService) { @ApiOperation("Returns records for the NFT") @GetMapping("/scope/{addr}/records") fun getNftRecords(@PathVariable addr: String) = ResponseEntity.ok(nftService.getRecordsForScope(addr)) + + @ApiOperation("Returns the scope spec as a json object") + @GetMapping("/scopeSpec/{scopeSpec}/json") + fun getScopeSpecJson(@PathVariable scopeSpec: String) = ResponseEntity.ok(nftService.getScopeSpecJson(scopeSpec)) + + @ApiOperation("Returns the contract spec as a json object") + @GetMapping("/contractSpec/{contractSpec}/json") + fun getContractSpecJson(@PathVariable contractSpec: String) = ResponseEntity.ok(nftService.getContractSpecJson(contractSpec)) + + @ApiOperation("Returns the record spec as a json object") + @GetMapping("/recordSpec/{recordSpec}/json") + fun getRecordSpecJson(@PathVariable recordSpec: String) = ResponseEntity.ok(nftService.getRecordSpecJson(recordSpec)) } diff --git a/service/src/main/kotlin/io/provenance/explorer/web/v2/SmartContractController.kt b/service/src/main/kotlin/io/provenance/explorer/web/v2/SmartContractController.kt index 14194b1a..fff92dae 100644 --- a/service/src/main/kotlin/io/provenance/explorer/web/v2/SmartContractController.kt +++ b/service/src/main/kotlin/io/provenance/explorer/web/v2/SmartContractController.kt @@ -23,6 +23,13 @@ import javax.validation.constraints.Min ) class SmartContractController(private val scService: SmartContractService) { + @ApiOperation("Returns paginated list of smart contract codes, code ID descending") + @GetMapping("/codes/all") + fun getCodesList( + @RequestParam(required = false, defaultValue = "1") @Min(1) page: Int, + @RequestParam(required = false, defaultValue = "10") @Min(1) count: Int + ) = ResponseEntity.ok(scService.getAllScCodesPaginated(page, count)) + @ApiOperation("Returns a smart contract code object") @GetMapping("/code/{id}") fun getCode(@PathVariable id: Int) = ResponseEntity.ok(scService.getCode(id)) @@ -35,7 +42,7 @@ class SmartContractController(private val scService: SmartContractService) { @RequestParam(required = false, defaultValue = "10") @Min(1) count: Int ) = ResponseEntity.ok(scService.getContractsByCode(id, page, count)) - @ApiOperation("Returns paginated list of smart contract codes, code ID descending") + @ApiOperation("Returns paginated list of smart contracts, creation date descending") @GetMapping("/contract/all") fun getContractsList( @RequestParam(required = false, defaultValue = "1") @Min(1) page: Int, diff --git a/service/src/main/kotlin/io/provenance/explorer/web/v2/UtilityController.kt b/service/src/main/kotlin/io/provenance/explorer/web/v2/UtilityController.kt index 7f1bbd6d..3e1c1a31 100644 --- a/service/src/main/kotlin/io/provenance/explorer/web/v2/UtilityController.kt +++ b/service/src/main/kotlin/io/provenance/explorer/web/v2/UtilityController.kt @@ -16,7 +16,7 @@ import org.springframework.web.bind.annotation.RestController @Validated @RestController -@RequestMapping(path = ["/api/v2"], produces = [MediaType.APPLICATION_JSON_VALUE]) +@RequestMapping(path = ["/api/v2/utility"], produces = [MediaType.APPLICATION_JSON_VALUE]) @Api( value = "Utility controller", produces = "application/json", consumes = "application/json", tags = ["Utilities"], description = "This should not be used by the UI" diff --git a/service/src/main/kotlin/io/provenance/explorer/web/v2/ValidatorController.kt b/service/src/main/kotlin/io/provenance/explorer/web/v2/ValidatorController.kt index f1fc0906..c2083bea 100644 --- a/service/src/main/kotlin/io/provenance/explorer/web/v2/ValidatorController.kt +++ b/service/src/main/kotlin/io/provenance/explorer/web/v2/ValidatorController.kt @@ -1,5 +1,6 @@ package io.provenance.explorer.web.v2 +import io.provenance.explorer.domain.models.explorer.Timeframe import io.provenance.explorer.service.ExplorerService import io.provenance.explorer.service.ValidatorService import io.swagger.annotations.Api @@ -80,4 +81,17 @@ class ValidatorController(private val validatorService: ValidatorService, privat @PathVariable id: String, @RequestParam(required = false, defaultValue = "100") blockCount: Int ) = ResponseEntity.ok(validatorService.getBlockLatencyData(id, blockCount)) + + @ApiOperation("Returns distinct validators with missed blocks for the timeframe") + @GetMapping("/missed_blocks/distinct") + fun missedBlocksDistinct( + @RequestParam(required = false, defaultValue = "HOUR") timeframe: Timeframe + ) = ResponseEntity.ok(validatorService.getDistinctValidatorsWithMissedBlocksInTimeframe(timeframe)) + + @ApiOperation("Returns validators with missed blocks for the timeframe") + @GetMapping("/missed_blocks") + fun missedBlocks( + @RequestParam(required = false, defaultValue = "HOUR") timeframe: Timeframe, + @RequestParam(required = false) validatorAddr: String?, + ) = ResponseEntity.ok(validatorService.getMissedBlocksForValidatorInTimeframe(timeframe, validatorAddr)) }