Skip to content

Commit

Permalink
Cherrypick/3343 3346 (#3349)
Browse files Browse the repository at this point in the history 
* reserved zk counters (#3343)

* reserved zk counters

* new prover image

* udpate counters names

* udpate counters names

* add gas used to reserved zkCounters

* add ReservedZKCounters to pool trasactions

* Add reserved zk counters to pool transactions (#3346)

* add reserved zk counters to pool transaction

* add reserved zk counters to pool transaction

* correct migration test
  • Loading branch information
@ToniRamirezM
ToniRamirezM authored Feb 22, 2024
1 parent 0204aff commit 49e837a
Show file tree
Hide file tree
Showing 27 changed files with 1,272 additions and 928 deletions.
7 changes: 7 additions & 0 deletions db/migrations/pool/0013.sql
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
-- +migrate Up
ALTER TABLE pool.transaction
ADD COLUMN reserved_zkcounters jsonb;

-- +migrate Down
ALTER TABLE pool.transaction
DROP COLUMN reserved_zkcounters;
50 changes: 50 additions & 0 deletions db/migrations/pool/0013_test.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
package pool_migrations_test

import (
"database/sql"
"testing"

"github.com/0xPolygonHermez/zkevm-node/state"
"github.com/stretchr/testify/require"
)

// this migration adds reserved_zkcounters to the transaction
type migrationTest0013 struct{}

func (m migrationTest0013) InsertData(db *sql.DB) error {
return nil
}

func (m migrationTest0013) RunAssertsAfterMigrationUp(t *testing.T, db *sql.DB) {
var reserved_zkcounters = state.ZKCounters{
GasUsed: 0,
KeccakHashes: 1,
PoseidonHashes: 2,
}

const insertTx = `
INSERT INTO pool.transaction (hash, ip, received_at, from_address, reserved_zkcounters)
VALUES ('0x0001', '127.0.0.1', '2023-12-07', '0x0011', $1)`

_, err := db.Exec(insertTx, reserved_zkcounters)
require.NoError(t, err)
}

func (m migrationTest0013) RunAssertsAfterMigrationDown(t *testing.T, db *sql.DB) {
var reserved_zkcounters = state.ZKCounters{
GasUsed: 0,
KeccakHashes: 1,
PoseidonHashes: 2,
}

const insertTx = `
INSERT INTO pool.transaction (hash, ip, received_at, from_address, reserved_zkcounters)
VALUES ('0x0001', '127.0.0.1', '2023-12-07', '0x0011', $1)`

_, err := db.Exec(insertTx, reserved_zkcounters)
require.Error(t, err)
}

func TestMigration0013(t *testing.T) {
runMigrationTest(t, 13, migrationTest0013{})
}
2 changes: 1 addition & 1 deletion docker-compose.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,7 @@ services:
zkevm-prover:
container_name: zkevm-prover
restart: unless-stopped
image: hermeznetwork/zkevm-prover:v5.0.0-RC4
image: hermeznetwork/zkevm-prover:v5.0.0-RC5
depends_on:
zkevm-state-db:
condition: service_healthy
Expand Down
36 changes: 18 additions & 18 deletions pool/config_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,30 +27,30 @@ func TestIsWithinConstraints(t *testing.T) {
{
desc: "All constraints within limits",
counters: state.ZKCounters{
GasUsed: 300,
UsedKeccakHashes: 50,
UsedPoseidonHashes: 100,
UsedPoseidonPaddings: 75,
UsedMemAligns: 500,
UsedArithmetics: 1000,
UsedBinaries: 2000,
UsedSteps: 2000,
UsedSha256Hashes_V2: 4000,
GasUsed: 300,
KeccakHashes: 50,
PoseidonHashes: 100,
PoseidonPaddings: 75,
MemAligns: 500,
Arithmetics: 1000,
Binaries: 2000,
Steps: 2000,
Sha256Hashes_V2: 4000,
},
expected: true,
},
{
desc: "All constraints exceed limits",
counters: state.ZKCounters{
GasUsed: 600,
UsedKeccakHashes: 150,
UsedPoseidonHashes: 300,
UsedPoseidonPaddings: 200,
UsedMemAligns: 2000,
UsedArithmetics: 3000,
UsedBinaries: 4000,
UsedSteps: 5000,
UsedSha256Hashes_V2: 6000,
GasUsed: 600,
KeccakHashes: 150,
PoseidonHashes: 300,
PoseidonPaddings: 200,
MemAligns: 2000,
Arithmetics: 3000,
Binaries: 4000,
Steps: 5000,
Sha256Hashes_V2: 6000,
},
expected: false,
},
Expand Down
2 changes: 1 addition & 1 deletion pool/interfaces.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ type storage interface {
GetTxFromAddressFromByHash(ctx context.Context, hash common.Hash) (common.Address, uint64, error)
GetTransactionByHash(ctx context.Context, hash common.Hash) (*Transaction, error)
GetTransactionByL2Hash(ctx context.Context, hash common.Hash) (*Transaction, error)
GetTxZkCountersByHash(ctx context.Context, hash common.Hash) (*state.ZKCounters, error)
GetTxZkCountersByHash(ctx context.Context, hash common.Hash) (*state.ZKCounters, *state.ZKCounters, error)
DeleteTransactionByHash(ctx context.Context, hash common.Hash) error
MarkWIPTxsAsPending(ctx context.Context) error
GetAllAddressesBlocked(ctx context.Context) ([]common.Address, error)
Expand Down
105 changes: 58 additions & 47 deletions pool/pgpoolstorage/pgpoolstorage.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,10 +75,11 @@ func (p *PostgresPoolStorage) AddTx(ctx context.Context, tx pool.Transaction) er
from_address,
is_wip,
ip,
failed_reason
failed_reason,
reserved_zkcounters
)
VALUES
($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, NULL)
($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, NULL, $20)
ON CONFLICT (hash) DO UPDATE SET
encoded = $2,
decoded = $3,
Expand All @@ -98,7 +99,8 @@ func (p *PostgresPoolStorage) AddTx(ctx context.Context, tx pool.Transaction) er
from_address = $17,
is_wip = $18,
ip = $19,
failed_reason = NULL
failed_reason = NULL,
reserved_zkcounters = $20
`

// Get FromAddress from the JSON data
Expand All @@ -116,18 +118,19 @@ func (p *PostgresPoolStorage) AddTx(ctx context.Context, tx pool.Transaction) er
gasPrice,
nonce,
tx.GasUsed,
tx.UsedKeccakHashes,
tx.UsedPoseidonHashes,
tx.UsedPoseidonPaddings,
tx.UsedMemAligns,
tx.UsedArithmetics,
tx.UsedBinaries,
tx.UsedSteps,
tx.UsedSha256Hashes_V2,
tx.KeccakHashes,
tx.PoseidonHashes,
tx.PoseidonPaddings,
tx.MemAligns,
tx.Arithmetics,
tx.Binaries,
tx.Steps,
tx.Sha256Hashes_V2,
tx.ReceivedAt,
fromAddress,
tx.IsWIP,
tx.IP); err != nil {
tx.IP,
tx.ReservedZKCounters); err != nil {
return err
}
return nil
Expand All @@ -144,11 +147,11 @@ func (p *PostgresPoolStorage) GetTxsByStatus(ctx context.Context, status pool.Tx
)
if limit == 0 {
sql = `SELECT encoded, status, received_at, is_wip, ip, cumulative_gas_used, used_keccak_hashes, used_poseidon_hashes, used_poseidon_paddings, used_mem_aligns,
used_arithmetics, used_binaries, used_steps, used_sha256_hashes, failed_reason FROM pool.transaction WHERE status = $1 ORDER BY gas_price DESC`
used_arithmetics, used_binaries, used_steps, used_sha256_hashes, failed_reason, reserved_zkcounters FROM pool.transaction WHERE status = $1 ORDER BY gas_price DESC`
rows, err = p.db.Query(ctx, sql, status.String())
} else {
sql = `SELECT encoded, status, received_at, is_wip, ip, cumulative_gas_used, used_keccak_hashes, used_poseidon_hashes, used_poseidon_paddings, used_mem_aligns,
used_arithmetics, used_binaries, used_steps, used_sha256_hashes, failed_reason FROM pool.transaction WHERE status = $1 ORDER BY gas_price DESC LIMIT $2`
used_arithmetics, used_binaries, used_steps, used_sha256_hashes, failed_reason, reserved_zkcounters FROM pool.transaction WHERE status = $1 ORDER BY gas_price DESC LIMIT $2`
rows, err = p.db.Query(ctx, sql, status.String(), limit)
}
if err != nil {
Expand Down Expand Up @@ -177,7 +180,7 @@ func (p *PostgresPoolStorage) GetNonWIPPendingTxs(ctx context.Context) ([]pool.T
)

sql = `SELECT encoded, status, received_at, is_wip, ip, cumulative_gas_used, used_keccak_hashes, used_poseidon_hashes, used_poseidon_paddings, used_mem_aligns,
used_arithmetics, used_binaries, used_steps, used_sha256_hashes, failed_reason FROM pool.transaction WHERE is_wip IS FALSE and status = $1`
used_arithmetics, used_binaries, used_steps, used_sha256_hashes, failed_reason, reserved_zkcounters FROM pool.transaction WHERE is_wip IS FALSE and status = $1`
rows, err = p.db.Query(ctx, sql, pool.TxStatusPending)

if err != nil {
Expand Down Expand Up @@ -236,7 +239,8 @@ func (p *PostgresPoolStorage) GetTxs(ctx context.Context, filterStatus pool.TxSt
received_at,
nonce,
is_wip,
ip
ip,
reserved_zkcounters
FROM
pool.transaction p1
WHERE
Expand Down Expand Up @@ -265,7 +269,8 @@ func (p *PostgresPoolStorage) GetTxs(ctx context.Context, filterStatus pool.TxSt
received_at,
nonce,
is_wip,
ip
ip,
reserved_zkcounters
FROM
pool.transaction p1
WHERE
Expand All @@ -285,8 +290,9 @@ func (p *PostgresPoolStorage) GetTxs(ctx context.Context, filterStatus pool.TxSt
cumulativeGasUsed uint64
usedKeccakHashes, usedPoseidonHashes, usedPoseidonPaddings,
usedMemAligns, usedArithmetics, usedBinaries, usedSteps, usedSHA256Hashes uint32
nonce uint64
isWIP bool
nonce uint64
isWIP bool
reservedZKCounters state.ZKCounters
)

args := []interface{}{filterStatus, minGasPrice, limit}
Expand Down Expand Up @@ -316,6 +322,7 @@ func (p *PostgresPoolStorage) GetTxs(ctx context.Context, filterStatus pool.TxSt
&nonce,
&isWIP,
&ip,
&reservedZKCounters,
)

if err != nil {
Expand All @@ -333,18 +340,19 @@ func (p *PostgresPoolStorage) GetTxs(ctx context.Context, filterStatus pool.TxSt
tx.Status = pool.TxStatus(status)
tx.ReceivedAt = receivedAt
tx.ZKCounters = state.ZKCounters{
GasUsed: cumulativeGasUsed,
UsedKeccakHashes: usedKeccakHashes,
UsedPoseidonHashes: usedPoseidonHashes,
UsedPoseidonPaddings: usedPoseidonPaddings,
UsedMemAligns: usedMemAligns,
UsedArithmetics: usedArithmetics,
UsedBinaries: usedBinaries,
UsedSteps: usedSteps,
UsedSha256Hashes_V2: usedSHA256Hashes,
GasUsed: cumulativeGasUsed,
KeccakHashes: usedKeccakHashes,
PoseidonHashes: usedPoseidonHashes,
PoseidonPaddings: usedPoseidonPaddings,
MemAligns: usedMemAligns,
Arithmetics: usedArithmetics,
Binaries: usedBinaries,
Steps: usedSteps,
Sha256Hashes_V2: usedSHA256Hashes,
}
tx.IsWIP = isWIP
tx.IP = ip
tx.ReservedZKCounters = reservedZKCounters
txs = append(txs, tx)
}

Expand Down Expand Up @@ -511,7 +519,7 @@ func (p *PostgresPoolStorage) IsTxPending(ctx context.Context, hash common.Hash)
// GetTxsByFromAndNonce get all the transactions from the pool with the same from and nonce
func (p *PostgresPoolStorage) GetTxsByFromAndNonce(ctx context.Context, from common.Address, nonce uint64) ([]pool.Transaction, error) {
sql := `SELECT encoded, status, received_at, is_wip, ip, cumulative_gas_used, used_keccak_hashes, used_poseidon_hashes,
used_poseidon_paddings, used_mem_aligns, used_arithmetics, used_binaries, used_steps, used_sha256_hashes, failed_reason
used_poseidon_paddings, used_mem_aligns, used_arithmetics, used_binaries, used_steps, used_sha256_hashes, failed_reason, reserved_zkcounters
FROM pool.transaction
WHERE from_address = $1
AND nonce = $2`
Expand Down Expand Up @@ -682,10 +690,11 @@ func scanTx(rows pgx.Rows) (*pool.Transaction, error) {
usedSteps uint32
usedSHA256Hashes uint32
failedReason *string
reservedZKCounters state.ZKCounters
)

if err := rows.Scan(&encoded, &status, &receivedAt, &isWIP, &ip, &cumulativeGasUsed, &usedKeccakHashes, &usedPoseidonHashes,
&usedPoseidonPaddings, &usedMemAligns, &usedArithmetics, &usedBinaries, &usedSteps, &usedSHA256Hashes, &failedReason); err != nil {
&usedPoseidonPaddings, &usedMemAligns, &usedArithmetics, &usedBinaries, &usedSteps, &usedSHA256Hashes, &failedReason, &reservedZKCounters); err != nil {
return nil, err
}

Expand All @@ -705,15 +714,16 @@ func scanTx(rows pgx.Rows) (*pool.Transaction, error) {
tx.IsWIP = isWIP
tx.IP = ip
tx.ZKCounters.GasUsed = cumulativeGasUsed
tx.ZKCounters.UsedKeccakHashes = usedKeccakHashes
tx.ZKCounters.UsedPoseidonHashes = usedPoseidonHashes
tx.ZKCounters.UsedPoseidonPaddings = usedPoseidonPaddings
tx.ZKCounters.UsedMemAligns = usedMemAligns
tx.ZKCounters.UsedArithmetics = usedArithmetics
tx.ZKCounters.UsedBinaries = usedBinaries
tx.ZKCounters.UsedSteps = usedSteps
tx.ZKCounters.UsedSha256Hashes_V2 = usedSHA256Hashes
tx.ZKCounters.KeccakHashes = usedKeccakHashes
tx.ZKCounters.PoseidonHashes = usedPoseidonHashes
tx.ZKCounters.PoseidonPaddings = usedPoseidonPaddings
tx.ZKCounters.MemAligns = usedMemAligns
tx.ZKCounters.Arithmetics = usedArithmetics
tx.ZKCounters.Binaries = usedBinaries
tx.ZKCounters.Steps = usedSteps
tx.ZKCounters.Sha256Hashes_V2 = usedSHA256Hashes
tx.FailedReason = failedReason
tx.ReservedZKCounters = reservedZKCounters

return tx, nil
}
Expand All @@ -728,21 +738,22 @@ func (p *PostgresPoolStorage) DeleteTransactionByHash(ctx context.Context, hash
}

// GetTxZkCountersByHash gets a transaction zkcounters by its hash
func (p *PostgresPoolStorage) GetTxZkCountersByHash(ctx context.Context, hash common.Hash) (*state.ZKCounters, error) {
var zkCounters state.ZKCounters
func (p *PostgresPoolStorage) GetTxZkCountersByHash(ctx context.Context, hash common.Hash) (*state.ZKCounters, *state.ZKCounters, error) {
var usedZKCounters state.ZKCounters
var reservedZKCounters state.ZKCounters

sql := `SELECT cumulative_gas_used, used_keccak_hashes, used_poseidon_hashes, used_poseidon_paddings, used_mem_aligns,
used_arithmetics, used_binaries, used_steps, used_sha256_hashes FROM pool.transaction WHERE hash = $1`
err := p.db.QueryRow(ctx, sql, hash.String()).Scan(&zkCounters.GasUsed, &zkCounters.UsedKeccakHashes,
&zkCounters.UsedPoseidonHashes, &zkCounters.UsedPoseidonPaddings,
&zkCounters.UsedMemAligns, &zkCounters.UsedArithmetics, &zkCounters.UsedBinaries, &zkCounters.UsedSteps, &zkCounters.UsedSha256Hashes_V2)
used_arithmetics, used_binaries, used_steps, used_sha256_hashes, reserved_zkcounters FROM pool.transaction WHERE hash = $1`
err := p.db.QueryRow(ctx, sql, hash.String()).Scan(&usedZKCounters.GasUsed, &usedZKCounters.KeccakHashes,
&usedZKCounters.PoseidonHashes, &usedZKCounters.PoseidonPaddings,
&usedZKCounters.MemAligns, &usedZKCounters.Arithmetics, &usedZKCounters.Binaries, &usedZKCounters.Steps, &usedZKCounters.Sha256Hashes_V2, &reservedZKCounters)
if errors.Is(err, pgx.ErrNoRows) {
return nil, pool.ErrNotFound
return nil, nil, pool.ErrNotFound
} else if err != nil {
return nil, err
return nil, nil, err
}

return &zkCounters, nil
return &usedZKCounters, &reservedZKCounters, nil
}

// MarkWIPTxsAsPending updates WIP status to non WIP
Expand Down
14 changes: 9 additions & 5 deletions pool/pool.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,8 @@ type Pool struct {
}

type preExecutionResponse struct {
usedZkCounters state.ZKCounters
usedZKCounters state.ZKCounters
reservedZKCounters state.ZKCounters
isExecutorLevelError bool
OOCError error
OOGError error
Expand Down Expand Up @@ -238,8 +239,9 @@ func (p *Pool) StoreTx(ctx context.Context, tx types.Transaction, ip string, isW
}

poolTx := NewTransaction(tx, ip, isWIP)
poolTx.ZKCounters = preExecutionResponse.usedZkCounters
poolTx.GasUsed = preExecutionResponse.txResponse.GasUsed
poolTx.ZKCounters = preExecutionResponse.usedZKCounters
poolTx.ReservedZKCounters = preExecutionResponse.reservedZKCounters

return p.storage.AddTx(ctx, *poolTx)
}
Expand Down Expand Up @@ -293,7 +295,7 @@ func (p *Pool) ValidateBreakEvenGasPrice(ctx context.Context, tx types.Transacti

// preExecuteTx executes a transaction to calculate its zkCounters
func (p *Pool) preExecuteTx(ctx context.Context, tx types.Transaction) (preExecutionResponse, error) {
response := preExecutionResponse{usedZkCounters: state.ZKCounters{}, OOCError: nil, OOGError: nil, isReverted: false}
response := preExecutionResponse{usedZKCounters: state.ZKCounters{}, reservedZKCounters: state.ZKCounters{}, OOCError: nil, OOGError: nil, isReverted: false}

// TODO: Add effectivePercentage = 0xFF to the request (factor of 1) when gRPC message is updated
processBatchResponse, err := p.state.PreProcessTransaction(ctx, &tx, nil)
Expand All @@ -310,7 +312,8 @@ func (p *Pool) preExecuteTx(ctx context.Context, tx types.Transaction) (preExecu
response.OOGError = err
}
if processBatchResponse != nil && processBatchResponse.BlockResponses != nil && len(processBatchResponse.BlockResponses) > 0 {
response.usedZkCounters = processBatchResponse.UsedZkCounters
response.usedZKCounters = processBatchResponse.UsedZkCounters
response.reservedZKCounters = processBatchResponse.ReservedZkCounters
response.txResponse = processBatchResponse.BlockResponses[0].TransactionResponses[0]
}
return response, nil
Expand All @@ -336,7 +339,8 @@ func (p *Pool) preExecuteTx(ctx context.Context, tx types.Transaction) (preExecu
}
}

response.usedZkCounters = processBatchResponse.UsedZkCounters
response.usedZKCounters = processBatchResponse.UsedZkCounters
response.reservedZKCounters = processBatchResponse.ReservedZkCounters
response.txResponse = processBatchResponse.BlockResponses[0].TransactionResponses[0]
}

Expand Down
Loading

0 comments on commit 49e837a

Please sign in to comment.