Handle deneb blobs.

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.20-bullseye as builder
+FROM golang:1.20-bookworm as builder
 
 WORKDIR /app
 
@@ -10,7 +10,7 @@ COPY . .
 
 RUN go build
 
-FROM debian:bullseye-slim
+FROM debian:bookworm-slim
 
 RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt install -y ca-certificates && apt-get clean && rm -rf /var/lib/apt/lists/*
 

clients.go

@@ -18,7 +18,7 @@ import (
 	"sync"
 
 	eth2client "github.com/attestantio/go-eth2-client"
-	autoclient "github.com/attestantio/go-eth2-client/auto"
+	"github.com/attestantio/go-eth2-client/http"
 	"github.com/pkg/errors"
 	"github.com/spf13/viper"
 	"github.com/wealdtech/chaind/util"
@@ -41,10 +41,10 @@ func fetchClient(ctx context.Context, address string) (eth2client.Service, error
 	var exists bool
 	if client, exists = clients[address]; !exists {
 		var err error
-		client, err = autoclient.New(ctx,
-			autoclient.WithLogLevel(util.LogLevel("eth2client")),
-			autoclient.WithTimeout(viper.GetDuration("eth2client.timeout")),
-			autoclient.WithAddress(address))
+		client, err = http.New(ctx,
+			http.WithLogLevel(util.LogLevel("eth2client")),
+			http.WithTimeout(viper.GetDuration("eth2client.timeout")),
+			http.WithAddress(address))
 		if err != nil {
 			return nil, errors.Wrap(err, "failed to initiate client")
 		}

go.mod

@@ -3,7 +3,7 @@ module github.com/wealdtech/chaind
 go 1.20
 
 require (
-	github.com/attestantio/go-eth2-client v0.19.5
+	github.com/attestantio/go-eth2-client v0.19.8
 	github.com/aws/aws-sdk-go v1.47.10
 	github.com/jackc/pgx-shopspring-decimal v0.0.0-20220624020537-1d36b5a1853e
 	github.com/jackc/pgx-zerolog v0.0.0-20230315001418-f978528409eb
@@ -51,7 +51,7 @@ require (
 	github.com/googleapis/gax-go/v2 v2.12.0 // indirect
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.1 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
-	github.com/holiman/uint256 v1.2.3 // indirect
+	github.com/holiman/uint256 v1.2.4 // indirect
 	github.com/huandu/go-clone v1.7.0 // indirect
 	github.com/jackc/pgpassfile v1.0.0 // indirect
 	github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
@@ -81,11 +81,11 @@ require (
 	go.opentelemetry.io/otel/metric v1.20.0 // indirect
 	go.opentelemetry.io/proto/otlp v1.0.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
-	golang.org/x/crypto v0.15.0 // indirect
+	golang.org/x/crypto v0.18.0 // indirect
 	golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa // indirect
 	golang.org/x/net v0.18.0 // indirect
 	golang.org/x/oauth2 v0.14.0 // indirect
-	golang.org/x/sys v0.14.0 // indirect
+	golang.org/x/sys v0.16.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 	golang.org/x/time v0.4.0 // indirect
 	golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect

go.sum

@@ -74,8 +74,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
-github.com/attestantio/go-eth2-client v0.19.5 h1:4V+vhXsCYji5jWrlONbr03GV7qoLRdzq96dLgXaqmek=
-github.com/attestantio/go-eth2-client v0.19.5/go.mod h1:mZve1kV9Ctj0I1HH9gdg+MnI8lZ+Cb2EktEtOYrBlsM=
+github.com/attestantio/go-eth2-client v0.19.8 h1:7hn5X1MohW+wpfnwOpysPpjSktZwTB0A2ClohOCgZLI=
+github.com/attestantio/go-eth2-client v0.19.8/go.mod h1:mZve1kV9Ctj0I1HH9gdg+MnI8lZ+Cb2EktEtOYrBlsM=
 github.com/aws/aws-sdk-go v1.44.81/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
 github.com/aws/aws-sdk-go v1.47.10 h1:cvufN7WkD1nlOgpRopsmxKQlFp5X1MfyAw4r7BBORQc=
 github.com/aws/aws-sdk-go v1.47.10/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk=
@@ -238,8 +238,8 @@ github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
-github.com/holiman/uint256 v1.2.3 h1:K8UWO1HUJpRMXBxbmaY1Y8IAMZC/RsKB+ArEnnK4l5o=
-github.com/holiman/uint256 v1.2.3/go.mod h1:SC8Ryt4n+UBbPbIBKaG9zbbDlp4jOru9xFZmPzLUTxw=
+github.com/holiman/uint256 v1.2.4 h1:jUc4Nk8fm9jZabQuqr2JzednajVmBpC+oiTiXZJEApU=
+github.com/holiman/uint256 v1.2.4/go.mod h1:EOMSn4q6Nyt9P6efbI3bueV4e1b3dGlUCXeiRV4ng7E=
 github.com/huandu/go-assert v1.1.5 h1:fjemmA7sSfYHJD7CUqs9qTwwfdNAx7/j2/ZlHXzNB3c=
 github.com/huandu/go-assert v1.1.5/go.mod h1:yOLvuqZwmcHIC5rIzrBhT7D3Q9c3GFnd0JrPVhn/06U=
 github.com/huandu/go-clone v1.7.0 h1:N+U/hh2bKrHmlzcBrIl68pQVbojbz086LqzmB87c/pY=
@@ -405,8 +405,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh
 golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
-golang.org/x/crypto v0.15.0 h1:frVn1TEaCEaZcn3Tmd7Y2b5KKPaZ+I32Q2OA3kYp5TA=
-golang.org/x/crypto v0.15.0/go.mod h1:4ChreQoLWfG3xLDer1WdlH5NdlQ3+mwnQq1YTKY+72g=
+golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc=
+golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -601,8 +601,8 @@ golang.org/x/sys v0.0.0-20220818161305-2296e01440c6/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q=
-golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU=
+golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

services/blocks/standard/handler.go

@@ -564,7 +564,7 @@ func (s *Service) updateBlobSidecarsForBlock(ctx context.Context,
 
 	dbBlobSidecars := make([]*chaindb.BlobSidecar, len(response.Data))
 	for i := range response.Data {
-		dbBlobSidecars[i] = s.dbBlobSidecar(ctx, response.Data[i])
+		dbBlobSidecars[i] = s.dbBlobSidecar(ctx, blockRoot, response.Data[i])
 	}
 
 	if err := s.blobSidecarsSetter.SetBlobSidecars(ctx, dbBlobSidecars); err != nil {
@@ -1179,13 +1179,16 @@ func (s *Service) beaconCommittee(ctx context.Context,
 }
 
 func (s *Service) dbBlobSidecar(_ context.Context,
+	blockRoot phase0.Root,
 	blobSidecar *deneb.BlobSidecar,
 ) *chaindb.BlobSidecar {
-	// TODO fix up.
 	return &chaindb.BlobSidecar{
-		Index:         blobSidecar.Index,
-		Blob:          blobSidecar.Blob,
-		KZGCommitment: blobSidecar.KZGCommitment,
-		KZGProof:      blobSidecar.KZGProof,
+		InclusionBlockRoot:          blockRoot,
+		InclusionSlot:               blobSidecar.SignedBlockHeader.Message.Slot,
+		InclusionIndex:              blobSidecar.Index,
+		Blob:                        blobSidecar.Blob,
+		KZGCommitment:               blobSidecar.KZGCommitment,
+		KZGProof:                    blobSidecar.KZGProof,
+		KZGCommitmentInclusionProof: blobSidecar.KZGCommitmentInclusionProof,
 	}
 }

services/chaindb/postgresql/blobsidecars.go

@@ -50,13 +50,12 @@ func (s *Service) BlobSidecars(ctx context.Context,
 
 	queryBuilder.WriteString(`
 SELECT f_block_root
-      ,f_index
       ,f_slot
-      ,f_block_parent_root
-      ,f_proposer_index
+      ,f_index
       ,f_blob
       ,f_kzg_commitment
       ,f_kzg_proof
+      ,f_kzg_commitment_inclusion_proof
 FROM t_blob_sidecars`)
 
 	conditions := make([]string, 0)
@@ -124,36 +123,38 @@ LIMIT $%d`, len(queryVals)))
 	for rows.Next() {
 		blobSidecar := &chaindb.BlobSidecar{}
 		var blockRoot []byte
-		var blockParentRoot []byte
 		var blob []byte
 		var kzgCommitment []byte
 		var kzgProof []byte
+		var kzgCommitmentInclusionProof [][]byte
 		err := rows.Scan(
 			&blockRoot,
-			&blobSidecar.Index,
-			&blobSidecar.Slot,
-			&blockParentRoot,
+			&blobSidecar.InclusionSlot,
+			&blobSidecar.InclusionIndex,
 			&blob,
 			&kzgCommitment,
 			&kzgProof,
+			&kzgCommitmentInclusionProof,
 		)
 		if err != nil {
 			return nil, errors.Wrap(err, "failed to scan row")
 		}
-		copy(blobSidecar.BlockRoot[:], blockRoot)
-		copy(blobSidecar.BlockParentRoot[:], blockParentRoot)
+		copy(blobSidecar.InclusionBlockRoot[:], blockRoot)
 		copy(blobSidecar.Blob[:], blob)
 		copy(blobSidecar.KZGCommitment[:], kzgCommitment)
 		copy(blobSidecar.KZGProof[:], kzgProof)
+		for i := range kzgCommitmentInclusionProof {
+			copy(blobSidecar.KZGCommitmentInclusionProof[i][:], kzgCommitmentInclusionProof[i])
+		}
 		blobSidecars = append(blobSidecars, blobSidecar)
 	}
 
 	// Always return order of slot then index.
 	sort.Slice(blobSidecars, func(i int, j int) bool {
-		if blobSidecars[i].Slot != blobSidecars[j].Slot {
-			return blobSidecars[i].Slot < blobSidecars[j].Slot
+		if blobSidecars[i].InclusionSlot != blobSidecars[j].InclusionSlot {
+			return blobSidecars[i].InclusionSlot < blobSidecars[j].InclusionSlot
 		}
-		return blobSidecars[i].Index < blobSidecars[j].Index
+		return blobSidecars[i].InclusionIndex < blobSidecars[j].InclusionIndex
 	})
 
 	return blobSidecars, nil

services/chaindb/postgresql/setblobsidecar.go

@@ -38,35 +38,36 @@ func (s *Service) SetBlobSidecar(ctx context.Context, blobSidecar *chaindb.BlobS
 		blobBytes = bytes.TrimRight(blobBytes, string([]byte{0x00}))
 		blob = &blobBytes
 	}
+	kzgCommitmentInclusionProof := make([]byte, 0, 17*32)
+	for i := range blobSidecar.KZGCommitmentInclusionProof[:] {
+		kzgCommitmentInclusionProof = append(kzgCommitmentInclusionProof, blobSidecar.KZGCommitmentInclusionProof[i][:]...)
+	}
 
 	if _, err := tx.Exec(ctx, `
 INSERT INTO t_blob_sidecars(f_block_root
-                           ,f_index
                            ,f_slot
-                           ,f_block_parent_root
-                           ,f_proposer_index
+                           ,f_index
                            ,f_blob
                            ,f_kzg_commitment
                            ,f_kzg_proof
+                           ,f_kzg_commitment_inclusion_proof
 						   )
-VALUES($1,$2,$3,$4,$5,$6,$7,$8)
+VALUES($1,$2,$3,$4,$5,$6,$7)
 ON CONFLICT(f_block_root,f_index) DO
 UPDATE
 SET f_slot = excluded.f_slot
-   ,f_block_parent_root = excluded.f_block_parent_root
-   ,f_proposer_index = excluded.f_proposer_index
    ,f_blob = excluded.f_blob
    ,f_kzg_commitment = excluded.f_kzg_commitment
    ,f_kzg_proof = excluded.f_kzg_proof
+   ,f_kzg_commitment_inclusion_proof = excluded.f_kzg_commitment_inclusion_proof
 `,
-		blobSidecar.BlockRoot[:],
-		blobSidecar.Index,
-		blobSidecar.Slot,
-		blobSidecar.BlockParentRoot[:],
-		blobSidecar.ProposerIndex,
+		blobSidecar.InclusionBlockRoot[:],
+		blobSidecar.InclusionSlot,
+		blobSidecar.InclusionIndex,
 		blob,
 		blobSidecar.KZGCommitment[:],
 		blobSidecar.KZGProof[:],
+		kzgCommitmentInclusionProof,
 	); err != nil {
 		return err
 	}

services/chaindb/postgresql/setblobsidecars.go

@@ -43,13 +43,12 @@ func (s *Service) SetBlobSidecars(ctx context.Context, blobSidecars []*chaindb.B
 		pgx.Identifier{"t_blob_sidecars"},
 		[]string{
 			"f_block_root",
-			"f_index",
 			"f_slot",
-			"f_block_parent_root",
-			"f_proposer_index",
+			"f_index",
 			"f_blob",
 			"f_kzg_commitment",
 			"f_kzg_proof",
+			"f_kzg_commitment_inclusion_proof",
 		},
 		pgx.CopyFromSlice(len(blobSidecars), func(i int) ([]interface{}, error) {
 			var blob *[]byte
@@ -59,15 +58,19 @@ func (s *Service) SetBlobSidecars(ctx context.Context, blobSidecars []*chaindb.B
 				blobBytes = bytes.TrimRight(blobBytes, string([]byte{0x00}))
 				blob = &blobBytes
 			}
+			kzgCommitmentInclusionProof := make([]byte, 0, 17*32)
+			for j := range blobSidecars[i].KZGCommitmentInclusionProof[:] {
+				kzgCommitmentInclusionProof = append(kzgCommitmentInclusionProof, blobSidecars[i].KZGCommitmentInclusionProof[j][:]...)
+			}
+
 			return []interface{}{
-				blobSidecars[i].BlockRoot[:],
-				blobSidecars[i].Index,
-				blobSidecars[i].Slot,
-				blobSidecars[i].BlockParentRoot[:],
-				blobSidecars[i].ProposerIndex,
+				blobSidecars[i].InclusionBlockRoot[:],
+				blobSidecars[i].InclusionSlot,
+				blobSidecars[i].InclusionIndex,
 				blob,
 				blobSidecars[i].KZGCommitment[:],
 				blobSidecars[i].KZGProof[:],
+				kzgCommitmentInclusionProof,
 			}, nil
 		}))
 

services/chaindb/postgresql/upgrader.go

@@ -1187,6 +1187,18 @@ CREATE UNIQUE INDEX IF NOT EXISTS i_block_withdrawals_1 ON t_block_withdrawals(f
 CREATE INDEX IF NOT EXISTS i_block_withdrawals_2 ON t_block_withdrawals(f_block_number);
 CREATE INDEX IF NOT EXISTS i_block_withdrawals_3 ON t_block_withdrawals(f_validator_index);
 CREATE INDEX IF NOT EXISTS i_block_withdrawals_4 ON t_block_withdrawals(f_address);
+
+CREATE TABLE t_blob_sidecars (
+  f_block_root                     BYTEA NOT NULL REFERENCES t_blocks(f_root) ON DELETE CASCADE
+ ,f_slot                           BIGINT NOT NULL
+ ,f_index                          INTEGER NOT NULL
+ ,f_blob                           BYTEA NOT NULL
+ ,f_kzg_commitment                 BYTEA NOT NULL
+ ,f_kzg_proof                      BYTEA NOT NULL
+ ,f_kzg_commitment_inclusion_proof BYTEA NOT NULL
+);
+CREATE UNIQUE INDEX i_blob_sidecars_1 ON t_blob_sidecars(f_block_root,f_index);
+CREATE INDEX i_blob_sidecars_2 ON t_blob_sidecars(f_slot);
 `); err != nil {
 		cancel()
 		return errors.Wrap(err, "failed to create initial tables")
@@ -1777,14 +1789,13 @@ func addBlobSidecars(ctx context.Context, s *Service) error {
 
 	if _, err := tx.Exec(ctx, `
 CREATE TABLE t_blob_sidecars (
-  f_block_root        BYTEA NOT NULL REFERENCES t_blocks(f_root) ON DELETE CASCADE
- ,f_index             INTEGER NOT NULL
- ,f_slot              BIGINT NOT NULL
- ,f_block_parent_root BYTEA NOT NULL
- ,f_proposer_index    BIGINT NOT NULL
- ,f_blob              BYTEA
- ,f_kzg_commitment    BYTEA NOT NULL
- ,f_kzg_proof         BYTEA NOT NULL
+  f_block_root                     BYTEA NOT NULL REFERENCES t_blocks(f_root) ON DELETE CASCADE
+ ,f_slot                           BIGINT NOT NULL
+ ,f_index                          INTEGER NOT NULL
+ ,f_blob                           BYTEA NOT NULL
+ ,f_kzg_commitment                 BYTEA NOT NULL
+ ,f_kzg_proof                      BYTEA NOT NULL
+ ,f_kzg_commitment_inclusion_proof BYTEA NOT NULL
 )
 `); err != nil {
 		return errors.Wrap(err, "failed to create t_blob_sidecars")

services/chaindb/types.go

@@ -327,12 +327,11 @@ type Withdrawal struct {
 
 // BlobSidecar holds information about a blob sidecar for a block.
 type BlobSidecar struct {
-	BlockRoot       phase0.Root
-	Index           deneb.BlobIndex
-	Slot            phase0.Slot
-	BlockParentRoot phase0.Root
-	ProposerIndex   phase0.ValidatorIndex
-	Blob            deneb.Blob
-	KZGCommitment   deneb.KZGCommitment
-	KZGProof        deneb.KZGProof
+	InclusionBlockRoot          phase0.Root
+	InclusionSlot               phase0.Slot
+	InclusionIndex              deneb.BlobIndex
+	Blob                        deneb.Blob
+	KZGCommitment               deneb.KZGCommitment
+	KZGProof                    deneb.KZGProof
+	KZGCommitmentInclusionProof deneb.KZGCommitmentInclusionProof
 }

services/summarizer/standard/handler.go

@@ -199,8 +199,8 @@ func (s *Service) summarizeValidators(ctx context.Context, targetEpoch phase0.Ep
 	// is beyond our summarized epoch we truncate to the summarized value.
 	// However, if we don't have validator balances the summarizer won't run at all
 	// for epochs, so if the last epoch is 0 we continue.
-	if targetEpoch > firstEpoch && md.LastEpoch != 0 {
-		targetEpoch = firstEpoch
+	if targetEpoch > md.LastEpoch && md.LastEpoch > 0 {
+		targetEpoch = md.LastEpoch
 	}
 
 	// Limit the number of epochs summarised per pass, if we are also pruning.