Skip to content

Commit

Permalink
fix(dashboard): new clickhouse scheme changes
Browse files Browse the repository at this point in the history
  • Loading branch information
sasha-bitfly committed Dec 17, 2024
1 parent 2113fc6 commit c68ae68
Show file tree
Hide file tree
Showing 14 changed files with 302 additions and 181 deletions.
61 changes: 61 additions & 0 deletions .github/workflows/backend-publish-docker-staging.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
#
name: Backend docker image staging

# Configures this workflow to run every time a change is pushed to the branch called `release`.
on:
push:
branches:
- staging-2
- staging-3

# Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds.
env:
REGISTRY: ghcr.io
IMAGE_NAME: ${{ github.repository }}_backend

# There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu.
jobs:
build-and-push-image:
runs-on: ubuntu-latest
# Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job.
permissions:
contents: read
packages: write
#
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Set version
run: |
echo "BEACONCHAIN_VERSION=$(TZ=UTC0 git show --quiet --date='format-local:%Y%m%d%H%M%S' --format="%cd" $GITHUB_SHA)-$(git describe $GITHUB_SHA --always --tags)" >> "$GITHUB_ENV"
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
# Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here.
- name: Log in to the Container registry
uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
# This step uses [docker/metadata-action](https://github.com/docker/metadata-action#about) to extract tags and labels that will be applied to the specified image. The `id` "meta" allows the output of this step to be referenced in a subsequent step. The `images` value provides the base name for the tags and labels.
- name: Extract metadata (tags, labels) for Docker
id: meta
uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
with:
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
tags: |
${{ env.BEACONCHAIN_VERSION }}
type=ref,event=branch
# This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages.
# It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see "[Usage](https://github.com/docker/build-push-action#usage)" in the README of the `docker/build-push-action` repository.
# It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step.
- name: Build and push Docker image
uses: docker/[email protected]
with:
context: .
file: backend/Dockerfile
push: true
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache
cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache,mode=max
1 change: 0 additions & 1 deletion .github/workflows/backend-publish-docker.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ on:
branches:
- main
- staging
- staging-2

# Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds.
env:
Expand Down
64 changes: 64 additions & 0 deletions .github/workflows/frontend-publish-docker-staging.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
#
name: Frontend docker image staging

# Configures this workflow to run every time a change is pushed to the branch called `release`.
on:
push:
branches:
- staging-2
- staging-3

# Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds.
env:
REGISTRY: ghcr.io
IMAGE_NAME: ${{ github.repository }}_frontend

# There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu.
jobs:
build-and-push-image:
runs-on: ubuntu-latest
# Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job.
permissions:
contents: read
packages: write
#
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Set version
run: |
echo "BEACONCHAIN_VERSION=$(TZ=UTC0 git show --quiet --date='format-local:%Y%m%d%H%M%S' --format="%cd" $GITHUB_SHA)-$(git describe $GITHUB_SHA --always --tags)" >> "$GITHUB_ENV"
# Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here.
- name: Log in to the Container registry
uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
# This step uses [docker/metadata-action](https://github.com/docker/metadata-action#about) to extract tags and labels that will be applied to the specified image. The `id` "meta" allows the output of this step to be referenced in a subsequent step. The `images` value provides the base name for the tags and labels.
- name: Extract metadata (tags, labels) for Docker
id: meta
uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
with:
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
tags: |
${{ env.BEACONCHAIN_VERSION }}
type=ref,event=branch
# This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages.
# It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see "[Usage](https://github.com/docker/build-push-action#usage)" in the README of the `docker/build-push-action` repository.
# It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step.
- name: Build and push Docker image
uses: docker/[email protected]
with:
context: "{{defaultContext}}:frontend"
push: true
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache
cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache,mode=max
secrets: |
NPMRC_FILE=${{ secrets.NPMRC_FILE }}
build-args: |
BEACONCHAIN_VERSION=${{ env.BEACONCHAIN_VERSION }}
1 change: 0 additions & 1 deletion .github/workflows/frontend-publish-docker.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ on:
branches:
- main
- staging
- staging-2

# Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds.
env:
Expand Down
45 changes: 15 additions & 30 deletions backend/pkg/api/data_access/vdb_blocks.go
Original file line number Diff line number Diff line change
Expand Up @@ -361,37 +361,22 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das
Slot uint64 `db:"slot"`
ClReward decimal.NullDecimal `db:"cl_reward"`
}{}
if utils.Config.Chain.ClConfig.DepositChainID == 17000 {
clRewardsQuery := goqu.Dialect("postgres").
From(goqu.T("consensus_payloads")).
Select(
goqu.C("slot"),
goqu.L("cl_attestations_reward + cl_sync_aggregate_reward + cl_slashing_inclusion_reward AS cl_reward"),
).Where(goqu.C("slot").In(slots))
clRewardsQuerySql, args, err := clRewardsQuery.Prepared(true).ToSQL()
if err != nil {
return nil, nil, err
}
err = d.alloyReader.SelectContext(ctx, &clRewardsData, clRewardsQuerySql, args...)
if err != nil {
return nil, nil, err
}
} else {
clRewardsQuery := goqu.Dialect("postgres").
From(goqu.L("mainnet.validator_proposal_rewards_slot")).
Select(
goqu.C("slot"),
goqu.L("attestations_reward + sync_aggregate_reward + slasher_reward AS cl_reward"),
).Where(goqu.C("slot").In(slots))
clRewardsQuerySql, args, err := clRewardsQuery.Prepared(true).ToSQL()
if err != nil {
return nil, nil, err
}
err = d.clickhouseReader.SelectContext(ctx, &clRewardsData, clRewardsQuerySql, args...)
if err != nil {
return nil, nil, err
}

clRewardsQuery := goqu.Dialect("postgres").
From(goqu.L("validator_proposal_rewards_slot")).
Select(
goqu.C("slot"),
goqu.L("attestations_reward + sync_aggregate_reward + slasher_reward AS cl_reward"),
).Where(goqu.C("slot").In(slots))
clRewardsQuerySql, args, err := clRewardsQuery.Prepared(true).ToSQL()
if err != nil {
return nil, nil, err
}
err = d.clickhouseReader.SelectContext(ctx, &clRewardsData, clRewardsQuerySql, args...)
if err != nil {
return nil, nil, err
}

clRewards := make(map[uint64]decimal.NullDecimal)
for _, reward := range clRewardsData {
clRewards[reward.Slot] = reward.ClReward
Expand Down
56 changes: 28 additions & 28 deletions backend/pkg/api/data_access/vdb_rewards.go
Original file line number Diff line number Diff line change
Expand Up @@ -86,9 +86,9 @@ func (d *DataAccessService) GetValidatorDashboardRewards(ctx context.Context, da
With("validators", goqu.L("(SELECT validator_index as validator_index, group_id FROM users_val_dashboards_validators WHERE dashboard_id = ?)", dashboardId.Id)).
Select(
goqu.L("e.epoch"),
goqu.L(`SUM(COALESCE(e.attestations_reward, 0) + COALESCE(e.blocks_cl_reward, 0) + COALESCE(e.sync_rewards, 0)) AS cl_rewards`),
goqu.L(`SUM(COALESCE(e.attestations_reward, 0) + COALESCE(e.blocks_cl_reward, 0) + COALESCE(e.sync_reward, 0)) AS cl_rewards`),
goqu.L("SUM(COALESCE(e.attestations_scheduled, 0)) AS attestations_scheduled"),
goqu.L("SUM(COALESCE(e.attestations_executed, 0)) AS attestations_executed"),
goqu.L("SUM(COALESCE(e.attestations_observed, 0)) AS attestations_observed"),
goqu.L("SUM(COALESCE(e.blocks_scheduled, 0)) AS blocks_scheduled"),
goqu.L("SUM(COALESCE(e.blocks_proposed, 0)) AS blocks_proposed"),
goqu.L("SUM(COALESCE(e.sync_scheduled, 0)) AS sync_scheduled"),
Expand Down Expand Up @@ -300,7 +300,7 @@ func (d *DataAccessService) GetValidatorDashboardRewards(ctx context.Context, da
GroupId int64 `db:"result_group_id"`
ClRewards int64 `db:"cl_rewards"`
AttestationsScheduled uint64 `db:"attestations_scheduled"`
AttestationsExecuted uint64 `db:"attestations_executed"`
AttestationsExecuted uint64 `db:"attestations_observed"`
BlocksScheduled uint64 `db:"blocks_scheduled"`
BlocksProposed uint64 `db:"blocks_proposed"`
SyncScheduled uint64 `db:"sync_scheduled"`
Expand Down Expand Up @@ -539,15 +539,15 @@ func (d *DataAccessService) GetValidatorDashboardGroupRewards(ctx context.Contex
goqu.L("COALESCE(e.attestations_inactivity_reward, 0) AS attestations_inactivity_reward"),
goqu.L("COALESCE(e.attestations_inclusion_reward, 0) AS attestations_inclusion_reward"),
goqu.L("COALESCE(e.attestations_scheduled, 0) AS attestations_scheduled"),
goqu.L("COALESCE(e.attestation_head_executed, 0) AS attestation_head_executed"),
goqu.L("COALESCE(e.attestation_source_executed, 0) AS attestation_source_executed"),
goqu.L("COALESCE(e.attestation_target_executed, 0) AS attestation_target_executed"),
goqu.L("COALESCE(e.attestations_head_executed, 0) AS attestations_head_executed"),
goqu.L("COALESCE(e.attestations_source_executed, 0) AS attestations_source_executed"),
goqu.L("COALESCE(e.attestations_target_executed, 0) AS attestations_target_executed"),
goqu.L("COALESCE(e.blocks_scheduled, 0) AS blocks_scheduled"),
goqu.L("COALESCE(e.blocks_proposed, 0) AS blocks_proposed"),
goqu.L("COALESCE(e.blocks_cl_reward, 0) AS blocks_cl_reward"),
goqu.L("COALESCE(e.sync_scheduled, 0) AS sync_scheduled"),
goqu.L("COALESCE(e.sync_executed, 0) AS sync_executed"),
goqu.L("COALESCE(e.sync_rewards, 0) AS sync_rewards"),
goqu.L("COALESCE(e.sync_reward, 0) AS sync_reward"),
goqu.L("(CASE WHEN e.slashed THEN 1 ELSE 0 END) AS slashed_in_epoch"),
goqu.L("COALESCE(e.blocks_slashing_count, 0) AS slashed_amount"),
goqu.L("COALESCE(e.blocks_cl_slasher_reward, 0) AS slasher_reward"),
Expand Down Expand Up @@ -601,18 +601,18 @@ func (d *DataAccessService) GetValidatorDashboardGroupRewards(ctx context.Contex
AttestationInactivitytReward decimal.Decimal `db:"attestations_inactivity_reward"`
AttestationInclusionReward decimal.Decimal `db:"attestations_inclusion_reward"`

AttestationsScheduled int64 `db:"attestations_scheduled"`
AttestationHeadExecuted int64 `db:"attestation_head_executed"`
AttestationSourceExecuted int64 `db:"attestation_source_executed"`
AttestationTargetExecuted int64 `db:"attestation_target_executed"`
AttestationsScheduled int64 `db:"attestations_scheduled"`
AttestationsHeadExecuted int64 `db:"attestations_head_executed"`
AttestationsSourceExecuted int64 `db:"attestations_source_executed"`
AttestationsTargetExecuted int64 `db:"attestations_target_executed"`

BlocksScheduled uint32 `db:"blocks_scheduled"`
BlocksProposed uint32 `db:"blocks_proposed"`
BlocksClReward decimal.Decimal `db:"blocks_cl_reward"`

SyncScheduled uint32 `db:"sync_scheduled"`
SyncExecuted uint32 `db:"sync_executed"`
SyncRewards decimal.Decimal `db:"sync_rewards"`
SyncReward decimal.Decimal `db:"sync_reward"`

SlashedInEpoch bool `db:"slashed_in_epoch"`
SlashedAmount uint32 `db:"slashed_amount"`
Expand Down Expand Up @@ -662,16 +662,16 @@ func (d *DataAccessService) GetValidatorDashboardGroupRewards(ctx context.Contex

for _, entry := range queryResult {
ret.AttestationsHead.Income = ret.AttestationsHead.Income.Add(entry.AttestationHeadReward.Mul(gWei))
ret.AttestationsHead.StatusCount.Success += uint64(entry.AttestationHeadExecuted)
ret.AttestationsHead.StatusCount.Failed += uint64(entry.AttestationsScheduled) - uint64(entry.AttestationHeadExecuted)
ret.AttestationsHead.StatusCount.Success += uint64(entry.AttestationsHeadExecuted)
ret.AttestationsHead.StatusCount.Failed += uint64(entry.AttestationsScheduled) - uint64(entry.AttestationsHeadExecuted)

ret.AttestationsSource.Income = ret.AttestationsSource.Income.Add(entry.AttestationSourceReward.Mul(gWei))
ret.AttestationsSource.StatusCount.Success += uint64(entry.AttestationSourceExecuted)
ret.AttestationsSource.StatusCount.Failed += uint64(entry.AttestationsScheduled) - uint64(entry.AttestationSourceExecuted)
ret.AttestationsSource.StatusCount.Success += uint64(entry.AttestationsSourceExecuted)
ret.AttestationsSource.StatusCount.Failed += uint64(entry.AttestationsScheduled) - uint64(entry.AttestationsSourceExecuted)

ret.AttestationsTarget.Income = ret.AttestationsTarget.Income.Add(entry.AttestationTargetReward.Mul(gWei))
ret.AttestationsTarget.StatusCount.Success += uint64(entry.AttestationTargetExecuted)
ret.AttestationsTarget.StatusCount.Failed += uint64(entry.AttestationsScheduled) - uint64(entry.AttestationTargetExecuted)
ret.AttestationsTarget.StatusCount.Success += uint64(entry.AttestationsTargetExecuted)
ret.AttestationsTarget.StatusCount.Failed += uint64(entry.AttestationsScheduled) - uint64(entry.AttestationsTargetExecuted)

ret.Inactivity.Income = ret.Inactivity.Income.Add(entry.AttestationInactivitytReward.Mul(gWei))
if entry.AttestationInactivitytReward.LessThan(decimal.Zero) {
Expand All @@ -684,7 +684,7 @@ func (d *DataAccessService) GetValidatorDashboardGroupRewards(ctx context.Contex
ret.Proposal.StatusCount.Success += uint64(entry.BlocksProposed)
ret.Proposal.StatusCount.Failed += uint64(entry.BlocksScheduled) - uint64(entry.BlocksProposed)

ret.Sync.Income = ret.Sync.Income.Add(entry.SyncRewards.Mul(gWei))
ret.Sync.Income = ret.Sync.Income.Add(entry.SyncReward.Mul(gWei))
ret.Sync.StatusCount.Success += uint64(entry.SyncExecuted)
ret.Sync.StatusCount.Failed += uint64(entry.SyncScheduled) - uint64(entry.SyncExecuted)

Expand Down Expand Up @@ -725,7 +725,7 @@ func (d *DataAccessService) GetValidatorDashboardRewardsChart(ctx context.Contex
rewardsDs := goqu.Dialect("postgres").
Select(
goqu.L("e.epoch"),
goqu.L(`SUM(COALESCE(e.attestations_reward, 0) + COALESCE(e.blocks_cl_reward, 0) + COALESCE(e.sync_rewards, 0)) AS cl_rewards`)).
goqu.L(`SUM(COALESCE(e.attestations_reward, 0) + COALESCE(e.blocks_cl_reward, 0) + COALESCE(e.sync_reward, 0)) AS cl_rewards`)).
From(goqu.L("validator_dashboard_data_epoch e")).
With("validators", goqu.L("(SELECT validator_index as validator_index, group_id FROM users_val_dashboards_validators WHERE dashboard_id = ?)", dashboardId.Id)).
Where(goqu.L("e.epoch_timestamp >= fromUnixTimestamp(?)", utils.EpochToTime(startEpoch).Unix()))
Expand Down Expand Up @@ -958,15 +958,15 @@ func (d *DataAccessService) GetValidatorDashboardDuties(ctx context.Context, das
Select(
goqu.L("e.validator_index"),
goqu.L("COALESCE(e.attestations_scheduled, 0) AS attestations_scheduled"),
goqu.L("COALESCE(e.attestation_source_executed, 0) AS attestation_source_executed"),
goqu.L("COALESCE(e.attestations_source_executed, 0) AS attestations_source_executed"),
goqu.L("COALESCE(e.attestations_source_reward, 0) AS attestations_source_reward"),
goqu.L("COALESCE(e.attestation_target_executed, 0) AS attestation_target_executed"),
goqu.L("COALESCE(e.attestations_target_executed, 0) AS attestations_target_executed"),
goqu.L("COALESCE(e.attestations_target_reward, 0) AS attestations_target_reward"),
goqu.L("COALESCE(e.attestation_head_executed, 0) AS attestation_head_executed"),
goqu.L("COALESCE(e.attestations_head_executed, 0) AS attestations_head_executed"),
goqu.L("COALESCE(e.attestations_head_reward, 0) AS attestations_head_reward"),
goqu.L("COALESCE(e.sync_scheduled, 0) AS sync_scheduled"),
goqu.L("COALESCE(e.sync_executed, 0) AS sync_executed"),
goqu.L("COALESCE(e.sync_rewards, 0) AS sync_rewards"),
goqu.L("COALESCE(e.sync_reward, 0) AS sync_reward"),
goqu.L("e.slashed AS slashed_in_epoch"),
goqu.L("COALESCE(e.blocks_slashing_count, 0) AS slashed_amount"),
goqu.L("COALESCE(e.blocks_cl_slasher_reward, 0) AS slasher_reward"),
Expand Down Expand Up @@ -1044,15 +1044,15 @@ func (d *DataAccessService) GetValidatorDashboardDuties(ctx context.Context, das
queryResult := []struct {
ValidatorIndex uint64 `db:"validator_index"`
AttestationsScheduled uint64 `db:"attestations_scheduled"`
AttestationsSourceExecuted uint64 `db:"attestation_source_executed"`
AttestationsSourceExecuted uint64 `db:"attestations_source_executed"`
AttestationsSourceReward int64 `db:"attestations_source_reward"`
AttestationsTargetExecuted uint64 `db:"attestation_target_executed"`
AttestationsTargetExecuted uint64 `db:"attestations_target_executed"`
AttestationsTargetReward int64 `db:"attestations_target_reward"`
AttestationsHeadExecuted uint64 `db:"attestation_head_executed"`
AttestationsHeadExecuted uint64 `db:"attestations_head_executed"`
AttestationsHeadReward int64 `db:"attestations_head_reward"`
SyncScheduled uint64 `db:"sync_scheduled"`
SyncExecuted uint64 `db:"sync_executed"`
SyncRewards int64 `db:"sync_rewards"`
SyncRewards int64 `db:"sync_reward"`
SlashedInEpoch bool `db:"slashed_in_epoch"`
SlashedAmount uint64 `db:"slashed_amount"`
SlasherReward int64 `db:"slasher_reward"`
Expand Down
Loading

0 comments on commit c68ae68

Please sign in to comment.