Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

support/db: PoC: Use COPY instead of INSERT in BatchInsertBuilder #4094

Closed
Prev Previous commit
Next Next commit
Make sure lib/pq escapes json and column names properly
2opremio committed Nov 24, 2021

Verified

This commit was signed with the committer’s verified signature. The key has expired.
2opremio Alfonso Acosta
commit 35ac98dd4bf69ea2c40d227261347a91658dd984
Original file line number Diff line number Diff line change
@@ -51,9 +51,10 @@ func (i *effectBatchInsertBuilder) Add(
"history_account_id": accountID,
"address_muxed": muxedAccount,
"history_operation_id": operationID,
"\"order\"": order,
"order": order,
"type": effectType,
"details": details,
// we need to convert to string in order to make the lib/pq's COPY escaping happy
"details": string(details),
})
}

6 changes: 5 additions & 1 deletion services/horizon/internal/db2/history/liquidity_pools.go
Original file line number Diff line number Diff line change
@@ -36,7 +36,11 @@ type LiquidityPool struct {
type LiquidityPoolAssetReserves []LiquidityPoolAssetReserve

func (c LiquidityPoolAssetReserves) Value() (driver.Value, error) {
return json.Marshal(c)
b, err := json.Marshal(c)
if err != nil {
return nil, err
}
return string(b), err
}

func (c *LiquidityPoolAssetReserves) Scan(value interface{}) error {
12 changes: 10 additions & 2 deletions services/horizon/internal/db2/history/main.go
Original file line number Diff line number Diff line change
@@ -366,7 +366,11 @@ type ExpAssetStatAccounts struct {
}

func (e ExpAssetStatAccounts) Value() (driver.Value, error) {
return json.Marshal(e)
b, err := json.Marshal(e)
if err != nil {
return nil, err
}
return string(b), nil
}

func (e *ExpAssetStatAccounts) Scan(src interface{}) error {
@@ -403,7 +407,11 @@ type ExpAssetStatBalances struct {
}

func (e ExpAssetStatBalances) Value() (driver.Value, error) {
return json.Marshal(e)
b, err := json.Marshal(e)
if err != nil {
return nil, err
}
return string(b), nil
}

func (e *ExpAssetStatBalances) Scan(src interface{}) error {
Original file line number Diff line number Diff line change
@@ -51,11 +51,12 @@ func (i *operationBatchInsertBuilder) Add(
sourceAccountMuxed null.String,
) error {
return i.builder.Row(ctx, map[string]interface{}{
"id": id,
"transaction_id": transactionID,
"application_order": applicationOrder,
"type": operationType,
"details": details,
"id": id,
"transaction_id": transactionID,
"application_order": applicationOrder,
"type": operationType,
// we need to convert to string in order to make the lib/pq's COPY escaping happy
"details": string(details),
"source_account": sourceAccount,
"source_account_muxed": sourceAccountMuxed,
})
Original file line number Diff line number Diff line change
@@ -14,7 +14,7 @@ import (
// rows into the history_trades table
type InsertTrade struct {
HistoryOperationID int64 `db:"history_operation_id"`
Order int32 `db:"\"order\""`
Order int32 `db:"order"`
LedgerCloseTime time.Time `db:"ledger_closed_at"`

CounterAssetID int64 `db:"counter_asset_id"`
4 changes: 1 addition & 3 deletions services/horizon/internal/db2/history/transaction_test.go
Original file line number Diff line number Diff line change
@@ -231,9 +231,7 @@ func TestInsertTransactionDoesNotAllowDuplicateIndex(t *testing.T) {
tt.Assert.NoError(insertBuilder.Add(tt.Ctx, secondTransaction, sequence))
tt.Assert.EqualError(
insertBuilder.Exec(tt.Ctx),
"error adding values while inserting to history_transactions: "+
"exec failed: pq: duplicate key value violates unique constraint "+
"\"hs_transaction_by_id\"",
"pq: duplicate key value violates unique constraint \"hs_transaction_by_id\"",
)

ledger := Ledger{