From e7b916f95a82d9882af5ba5c42381a0821dde459 Mon Sep 17 00:00:00 2001 From: Daniel Mancia <21249320+dmanc@users.noreply.github.com> Date: Tue, 13 Aug 2024 21:18:51 -0700 Subject: [PATCH 1/5] Add an API to fetch blobs from a given batch header hash (#688) --- .../common/blobstore/blob_metadata_store.go | 92 +++++++ .../blobstore/blob_metadata_store_test.go | 101 +++++++- disperser/common/blobstore/shared_storage.go | 4 + .../common/blobstore/shared_storage_test.go | 228 ++++++++++++++++++ disperser/common/inmem/store.go | 42 ++++ disperser/dataapi/blobs_handlers.go | 119 +++++++++ disperser/dataapi/docs/docs.go | 61 +++++ disperser/dataapi/docs/swagger.json | 38 ++- disperser/dataapi/docs/swagger.yaml | 35 ++- disperser/dataapi/server.go | 198 +++++++++------ disperser/dataapi/server_test.go | 126 +++++++++- disperser/dataapi/utils.go | 2 +- disperser/disperser.go | 9 + 13 files changed, 929 insertions(+), 126 deletions(-) diff --git a/disperser/common/blobstore/blob_metadata_store.go b/disperser/common/blobstore/blob_metadata_store.go index 4a1ea381d8..033814bb79 100644 --- a/disperser/common/blobstore/blob_metadata_store.go +++ b/disperser/common/blobstore/blob_metadata_store.go @@ -130,6 +130,9 @@ func (s *BlobMetadataStore) GetBlobMetadataByStatusCount(ctx context.Context, st // GetBlobMetadataByStatusWithPagination returns all the metadata with the given status upto the specified limit // along with items, also returns a pagination token that can be used to fetch the next set of items +// +// Note that this may not return all the metadata for the batch if dynamodb query limit is reached. +// e.g 1mb limit for a single query func (s *BlobMetadataStore) GetBlobMetadataByStatusWithPagination(ctx context.Context, status disperser.BlobStatus, limit int32, exclusiveStartKey *disperser.BlobStoreExclusiveStartKey) ([]*disperser.BlobMetadata, *disperser.BlobStoreExclusiveStartKey, error) { var attributeMap map[string]types.AttributeValue @@ -203,6 +206,72 @@ func (s *BlobMetadataStore) GetAllBlobMetadataByBatch(ctx context.Context, batch return metadatas, nil } +// GetBlobMetadataByStatusWithPagination returns all the metadata with the given status upto the specified limit +// along with items, also returns a pagination token that can be used to fetch the next set of items +// +// Note that this may not return all the metadata for the batch if dynamodb query limit is reached. +// e.g 1mb limit for a single query +func (s *BlobMetadataStore) GetAllBlobMetadataByBatchWithPagination( + ctx context.Context, + batchHeaderHash [32]byte, + limit int32, + exclusiveStartKey *disperser.BatchIndexExclusiveStartKey, +) ([]*disperser.BlobMetadata, *disperser.BatchIndexExclusiveStartKey, error) { + var attributeMap map[string]types.AttributeValue + var err error + + // Convert the exclusive start key to a map of AttributeValue + if exclusiveStartKey != nil { + attributeMap, err = convertToAttribMapBatchIndex(exclusiveStartKey) + if err != nil { + return nil, nil, err + } + } + + queryResult, err := s.dynamoDBClient.QueryIndexWithPagination( + ctx, + s.tableName, + batchIndexName, + "BatchHeaderHash = :batch_header_hash", + commondynamodb.ExpresseionValues{ + ":batch_header_hash": &types.AttributeValueMemberB{ + Value: batchHeaderHash[:], + }, + }, + limit, + attributeMap, + ) + if err != nil { + return nil, nil, err + } + + s.logger.Info("Query result", "items", len(queryResult.Items), "lastEvaluatedKey", queryResult.LastEvaluatedKey) + // When no more results to fetch, the LastEvaluatedKey is nil + if queryResult.Items == nil && queryResult.LastEvaluatedKey == nil { + return nil, nil, nil + } + + metadata := make([]*disperser.BlobMetadata, len(queryResult.Items)) + for i, item := range queryResult.Items { + metadata[i], err = UnmarshalBlobMetadata(item) + if err != nil { + return nil, nil, err + } + } + + lastEvaluatedKey := queryResult.LastEvaluatedKey + if lastEvaluatedKey == nil { + return metadata, nil, nil + } + + // Convert the last evaluated key to a disperser.BatchIndexExclusiveStartKey + exclusiveStartKey, err = convertToExclusiveStartKeyBatchIndex(lastEvaluatedKey) + if err != nil { + return nil, nil, err + } + return metadata, exclusiveStartKey, nil +} + func (s *BlobMetadataStore) GetBlobMetadataInBatch(ctx context.Context, batchHeaderHash [32]byte, blobIndex uint32) (*disperser.BlobMetadata, error) { items, err := s.dynamoDBClient.QueryIndex(ctx, s.tableName, batchIndexName, "BatchHeaderHash = :batch_header_hash AND BlobIndex = :blob_index", commondynamodb.ExpresseionValues{ ":batch_header_hash": &types.AttributeValueMemberB{ @@ -468,6 +537,16 @@ func convertToExclusiveStartKey(exclusiveStartKeyMap map[string]types.AttributeV return &blobStoreExclusiveStartKey, nil } +func convertToExclusiveStartKeyBatchIndex(exclusiveStartKeyMap map[string]types.AttributeValue) (*disperser.BatchIndexExclusiveStartKey, error) { + blobStoreExclusiveStartKey := disperser.BatchIndexExclusiveStartKey{} + err := attributevalue.UnmarshalMap(exclusiveStartKeyMap, &blobStoreExclusiveStartKey) + if err != nil { + return nil, err + } + + return &blobStoreExclusiveStartKey, nil +} + func convertToAttribMap(blobStoreExclusiveStartKey *disperser.BlobStoreExclusiveStartKey) (map[string]types.AttributeValue, error) { if blobStoreExclusiveStartKey == nil { // Return an empty map or nil @@ -480,3 +559,16 @@ func convertToAttribMap(blobStoreExclusiveStartKey *disperser.BlobStoreExclusive } return avMap, nil } + +func convertToAttribMapBatchIndex(blobStoreExclusiveStartKey *disperser.BatchIndexExclusiveStartKey) (map[string]types.AttributeValue, error) { + if blobStoreExclusiveStartKey == nil { + // Return an empty map or nil + return nil, nil + } + + avMap, err := attributevalue.MarshalMap(blobStoreExclusiveStartKey) + if err != nil { + return nil, err + } + return avMap, nil +} diff --git a/disperser/common/blobstore/blob_metadata_store_test.go b/disperser/common/blobstore/blob_metadata_store_test.go index 4624619460..ab00338b31 100644 --- a/disperser/common/blobstore/blob_metadata_store_test.go +++ b/disperser/common/blobstore/blob_metadata_store_test.go @@ -88,7 +88,7 @@ func TestBlobMetadataStoreOperations(t *testing.T) { assert.NoError(t, err) assert.Equal(t, int32(1), finalizedCount) - confirmedMetadata := getConfirmedMetadata(t, blobKey1) + confirmedMetadata := getConfirmedMetadata(t, blobKey1, 1) err = blobMetadataStore.UpdateBlobMetadata(ctx, blobKey1, confirmedMetadata) assert.NoError(t, err) @@ -188,6 +188,102 @@ func TestBlobMetadataStoreOperationsWithPagination(t *testing.T) { }) } +func TestGetAllBlobMetadataByBatchWithPagination(t *testing.T) { + ctx := context.Background() + blobKey1 := disperser.BlobKey{ + BlobHash: blobHash, + MetadataHash: "hash", + } + metadata1 := &disperser.BlobMetadata{ + MetadataHash: blobKey1.MetadataHash, + BlobHash: blobHash, + BlobStatus: disperser.Processing, + Expiry: 0, + NumRetries: 0, + RequestMetadata: &disperser.RequestMetadata{ + BlobRequestHeader: blob.RequestHeader, + BlobSize: blobSize, + RequestedAt: 123, + }, + } + blobKey2 := disperser.BlobKey{ + BlobHash: "blob2", + MetadataHash: "hash2", + } + metadata2 := &disperser.BlobMetadata{ + MetadataHash: blobKey2.MetadataHash, + BlobHash: blobKey2.BlobHash, + BlobStatus: disperser.Finalized, + Expiry: 0, + NumRetries: 0, + RequestMetadata: &disperser.RequestMetadata{ + BlobRequestHeader: blob.RequestHeader, + BlobSize: blobSize, + RequestedAt: 123, + }, + ConfirmationInfo: &disperser.ConfirmationInfo{}, + } + err := blobMetadataStore.QueueNewBlobMetadata(ctx, metadata1) + assert.NoError(t, err) + err = blobMetadataStore.QueueNewBlobMetadata(ctx, metadata2) + assert.NoError(t, err) + + confirmedMetadata1 := getConfirmedMetadata(t, blobKey1, 1) + err = blobMetadataStore.UpdateBlobMetadata(ctx, blobKey1, confirmedMetadata1) + assert.NoError(t, err) + + confirmedMetadata2 := getConfirmedMetadata(t, blobKey2, 2) + err = blobMetadataStore.UpdateBlobMetadata(ctx, blobKey2, confirmedMetadata2) + assert.NoError(t, err) + + // Fetch the blob metadata with limit 1 + metadata, exclusiveStartKey, err := blobMetadataStore.GetAllBlobMetadataByBatchWithPagination(ctx, confirmedMetadata1.ConfirmationInfo.BatchHeaderHash, 1, nil) + assert.NoError(t, err) + assert.Equal(t, metadata[0], confirmedMetadata1) + assert.NotNil(t, exclusiveStartKey) + assert.Equal(t, confirmedMetadata1.ConfirmationInfo.BlobIndex, exclusiveStartKey.BlobIndex) + + // Get the next blob metadata with limit 1 and the exclusive start key + metadata, exclusiveStartKey, err = blobMetadataStore.GetAllBlobMetadataByBatchWithPagination(ctx, confirmedMetadata1.ConfirmationInfo.BatchHeaderHash, 1, exclusiveStartKey) + assert.NoError(t, err) + assert.Equal(t, metadata[0], confirmedMetadata2) + assert.Equal(t, confirmedMetadata2.ConfirmationInfo.BlobIndex, exclusiveStartKey.BlobIndex) + + // Fetching the next blob metadata should return an empty list + metadata, exclusiveStartKey, err = blobMetadataStore.GetAllBlobMetadataByBatchWithPagination(ctx, confirmedMetadata1.ConfirmationInfo.BatchHeaderHash, 1, exclusiveStartKey) + assert.NoError(t, err) + assert.Len(t, metadata, 0) + assert.Nil(t, exclusiveStartKey) + + // Fetch the blob metadata with limit 2 + metadata, exclusiveStartKey, err = blobMetadataStore.GetAllBlobMetadataByBatchWithPagination(ctx, confirmedMetadata1.ConfirmationInfo.BatchHeaderHash, 2, nil) + assert.NoError(t, err) + assert.Len(t, metadata, 2) + assert.Equal(t, metadata[0], confirmedMetadata1) + assert.Equal(t, metadata[1], confirmedMetadata2) + assert.NotNil(t, exclusiveStartKey) + assert.Equal(t, confirmedMetadata2.ConfirmationInfo.BlobIndex, exclusiveStartKey.BlobIndex) + + // Fetch the blob metadata with limit 3 should return only 2 items + metadata, exclusiveStartKey, err = blobMetadataStore.GetAllBlobMetadataByBatchWithPagination(ctx, confirmedMetadata1.ConfirmationInfo.BatchHeaderHash, 3, nil) + assert.NoError(t, err) + assert.Len(t, metadata, 2) + assert.Equal(t, metadata[0], confirmedMetadata1) + assert.Equal(t, metadata[1], confirmedMetadata2) + assert.Nil(t, exclusiveStartKey) + + deleteItems(t, []commondynamodb.Key{ + { + "MetadataHash": &types.AttributeValueMemberS{Value: blobKey1.MetadataHash}, + "BlobHash": &types.AttributeValueMemberS{Value: blobKey1.BlobHash}, + }, + { + "MetadataHash": &types.AttributeValueMemberS{Value: blobKey2.MetadataHash}, + "BlobHash": &types.AttributeValueMemberS{Value: blobKey2.BlobHash}, + }, + }) +} + func TestBlobMetadataStoreOperationsWithPaginationNoStoredBlob(t *testing.T) { ctx := context.Background() // Query BlobMetadataStore for a blob that does not exist @@ -255,9 +351,8 @@ func deleteItems(t *testing.T, keys []commondynamodb.Key) { assert.NoError(t, err) } -func getConfirmedMetadata(t *testing.T, metadataKey disperser.BlobKey) *disperser.BlobMetadata { +func getConfirmedMetadata(t *testing.T, metadataKey disperser.BlobKey, blobIndex uint32) *disperser.BlobMetadata { batchHeaderHash := [32]byte{1, 2, 3} - blobIndex := uint32(1) requestedAt := uint64(time.Now().Nanosecond()) var commitX, commitY fp.Element _, err := commitX.SetString("21661178944771197726808973281966770251114553549453983978976194544185382599016") diff --git a/disperser/common/blobstore/shared_storage.go b/disperser/common/blobstore/shared_storage.go index 1e8b7a45f2..456818a64b 100644 --- a/disperser/common/blobstore/shared_storage.go +++ b/disperser/common/blobstore/shared_storage.go @@ -242,6 +242,10 @@ func (s *SharedBlobStore) GetAllBlobMetadataByBatch(ctx context.Context, batchHe return s.blobMetadataStore.GetAllBlobMetadataByBatch(ctx, batchHeaderHash) } +func (s *SharedBlobStore) GetAllBlobMetadataByBatchWithPagination(ctx context.Context, batchHeaderHash [32]byte, limit int32, exclusiveStartKey *disperser.BatchIndexExclusiveStartKey) ([]*disperser.BlobMetadata, *disperser.BatchIndexExclusiveStartKey, error) { + return s.blobMetadataStore.GetAllBlobMetadataByBatchWithPagination(ctx, batchHeaderHash, limit, exclusiveStartKey) +} + // GetMetadata returns a blob metadata given a metadata key func (s *SharedBlobStore) GetBlobMetadata(ctx context.Context, metadataKey disperser.BlobKey) (*disperser.BlobMetadata, error) { return s.blobMetadataStore.GetBlobMetadata(ctx, metadataKey) diff --git a/disperser/common/blobstore/shared_storage_test.go b/disperser/common/blobstore/shared_storage_test.go index 6c0b438942..f423967d93 100644 --- a/disperser/common/blobstore/shared_storage_test.go +++ b/disperser/common/blobstore/shared_storage_test.go @@ -8,9 +8,11 @@ import ( "testing" "time" + commondynamodb "github.com/Layr-Labs/eigenda/common/aws/dynamodb" "github.com/Layr-Labs/eigenda/core" "github.com/Layr-Labs/eigenda/disperser" "github.com/Layr-Labs/eigenda/encoding" + "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" "github.com/stretchr/testify/assert" "github.com/ethereum/go-ethereum/common" @@ -174,6 +176,232 @@ func TestSharedBlobStore(t *testing.T) { assert.NotNil(t, blob2Metadata) assertMetadata(t, blobKey, blobSize, requestedAt, disperser.Finalized, blob1Metadata) assertMetadata(t, blobKey2, blobSize2, requestedAt, disperser.InsufficientSignatures, blob2Metadata) + + // Cleanup: Delete test items + t.Cleanup(func() { + deleteItems(t, []commondynamodb.Key{ + { + "MetadataHash": &types.AttributeValueMemberS{Value: blobKey.MetadataHash}, + "BlobHash": &types.AttributeValueMemberS{Value: blobKey.BlobHash}, + }, + { + "MetadataHash": &types.AttributeValueMemberS{Value: blobKey2.MetadataHash}, + "BlobHash": &types.AttributeValueMemberS{Value: blobKey2.BlobHash}, + }, + }) + }) +} + +func TestSharedBlobStoreBlobMetadataStoreOperationsWithPagination(t *testing.T) { + ctx := context.Background() + blobKey1 := disperser.BlobKey{ + BlobHash: blobHash, + MetadataHash: "hash", + } + metadata1 := &disperser.BlobMetadata{ + MetadataHash: blobKey1.MetadataHash, + BlobHash: blobHash, + BlobStatus: disperser.Processing, + Expiry: 0, + NumRetries: 0, + RequestMetadata: &disperser.RequestMetadata{ + BlobRequestHeader: blob.RequestHeader, + BlobSize: blobSize, + RequestedAt: 123, + }, + } + blobKey2 := disperser.BlobKey{ + BlobHash: "blob2", + MetadataHash: "hash2", + } + metadata2 := &disperser.BlobMetadata{ + MetadataHash: blobKey2.MetadataHash, + BlobHash: blobKey2.BlobHash, + BlobStatus: disperser.Finalized, + Expiry: 0, + NumRetries: 0, + RequestMetadata: &disperser.RequestMetadata{ + BlobRequestHeader: blob.RequestHeader, + BlobSize: blobSize, + RequestedAt: 123, + }, + ConfirmationInfo: &disperser.ConfirmationInfo{}, + } + + // Setup: Queue new blob metadata + err := blobMetadataStore.QueueNewBlobMetadata(ctx, metadata1) + assert.NoError(t, err) + err = blobMetadataStore.QueueNewBlobMetadata(ctx, metadata2) + assert.NoError(t, err) + + // Test: Fetch individual blob metadata + fetchedMetadata, err := sharedStorage.GetBlobMetadata(ctx, blobKey1) + assert.NoError(t, err) + assert.Equal(t, metadata1, fetchedMetadata) + fetchedMetadata, err = sharedStorage.GetBlobMetadata(ctx, blobKey2) + assert.NoError(t, err) + assert.Equal(t, metadata2, fetchedMetadata) + + // Test: Fetch blob metadata by status with pagination + t.Run("Fetch Processing Blobs", func(t *testing.T) { + processing, lastEvaluatedKey, err := sharedStorage.GetBlobMetadataByStatusWithPagination(ctx, disperser.Processing, 1, nil) + assert.NoError(t, err) + assert.Len(t, processing, 1) + assert.Equal(t, metadata1, processing[0]) + assert.NotNil(t, lastEvaluatedKey) + + // Fetch next page (should be empty) + nextProcessing, nextLastEvaluatedKey, err := sharedStorage.GetBlobMetadataByStatusWithPagination(ctx, disperser.Processing, 1, lastEvaluatedKey) + assert.NoError(t, err) + assert.Len(t, nextProcessing, 0) + assert.Nil(t, nextLastEvaluatedKey) + }) + + t.Run("Fetch Finalized Blobs", func(t *testing.T) { + finalized, lastEvaluatedKey, err := sharedStorage.GetBlobMetadataByStatusWithPagination(ctx, disperser.Finalized, 1, nil) + assert.NoError(t, err) + assert.Len(t, finalized, 1) + assert.Equal(t, metadata2, finalized[0]) + assert.NotNil(t, lastEvaluatedKey) + + // Fetch next page (should be empty) + nextFinalized, nextLastEvaluatedKey, err := sharedStorage.GetBlobMetadataByStatusWithPagination(ctx, disperser.Finalized, 1, lastEvaluatedKey) + assert.NoError(t, err) + assert.Len(t, nextFinalized, 0) + assert.Nil(t, nextLastEvaluatedKey) + }) + + // Cleanup: Delete test items + t.Cleanup(func() { + deleteItems(t, []commondynamodb.Key{ + { + "MetadataHash": &types.AttributeValueMemberS{Value: blobKey1.MetadataHash}, + "BlobHash": &types.AttributeValueMemberS{Value: blobKey1.BlobHash}, + }, + { + "MetadataHash": &types.AttributeValueMemberS{Value: blobKey2.MetadataHash}, + "BlobHash": &types.AttributeValueMemberS{Value: blobKey2.BlobHash}, + }, + }) + }) +} + +func TestSharedBlobStoreGetAllBlobMetadataByBatchWithPagination(t *testing.T) { + ctx := context.Background() + batchHeaderHash := [32]byte{1, 2, 3} + + // Create and store multiple blob metadata for the same batch + numBlobs := 5 + blobKeys := make([]disperser.BlobKey, numBlobs) + for i := 0; i < numBlobs; i++ { + blobKey := disperser.BlobKey{ + BlobHash: fmt.Sprintf("blob%d", i), + MetadataHash: fmt.Sprintf("hash%d", i), + } + blobKeys[i] = blobKey + + metadata := &disperser.BlobMetadata{ + BlobHash: blobKey.BlobHash, + MetadataHash: blobKey.MetadataHash, + BlobStatus: disperser.Confirmed, + RequestMetadata: &disperser.RequestMetadata{ + BlobRequestHeader: blob.RequestHeader, + BlobSize: blobSize, + RequestedAt: uint64(time.Now().UnixNano()), + }, + ConfirmationInfo: &disperser.ConfirmationInfo{ + BatchHeaderHash: batchHeaderHash, + BlobIndex: uint32(i), + }, + } + + err := blobMetadataStore.QueueNewBlobMetadata(ctx, metadata) + assert.NoError(t, err) + } + + // Test pagination with a page size of 2 + t.Run("Fetch All Blobs with Pagination", func(t *testing.T) { + var allFetchedMetadata []*disperser.BlobMetadata + var lastEvaluatedKey *disperser.BatchIndexExclusiveStartKey + pageSize := int32(2) + + for { + fetchedMetadata, newLastEvaluatedKey, err := sharedStorage.GetAllBlobMetadataByBatchWithPagination(ctx, batchHeaderHash, pageSize, lastEvaluatedKey) + assert.NoError(t, err) + + allFetchedMetadata = append(allFetchedMetadata, fetchedMetadata...) + + if newLastEvaluatedKey == nil { + assert.Len(t, fetchedMetadata, numBlobs%int(pageSize)) + break + } else { + assert.Len(t, fetchedMetadata, int(pageSize)) + } + lastEvaluatedKey = newLastEvaluatedKey + } + + assert.Len(t, allFetchedMetadata, numBlobs) + + // Verify that all blob metadata is fetched and in the correct order + for i, metadata := range allFetchedMetadata { + assert.Equal(t, fmt.Sprintf("blob%d", i), metadata.BlobHash) + assert.Equal(t, fmt.Sprintf("hash%d", i), metadata.MetadataHash) + assert.Equal(t, uint32(i), metadata.ConfirmationInfo.BlobIndex) + } + }) + + // Test pagination with a page size of 10 + t.Run("Fetch All Blobs with Pagination (Page Size > Num Blobs)", func(t *testing.T) { + var allFetchedMetadata []*disperser.BlobMetadata + var lastEvaluatedKey *disperser.BatchIndexExclusiveStartKey + pageSize := int32(10) + + for { + fetchedMetadata, newLastEvaluatedKey, err := sharedStorage.GetAllBlobMetadataByBatchWithPagination(ctx, batchHeaderHash, pageSize, lastEvaluatedKey) + assert.NoError(t, err) + + allFetchedMetadata = append(allFetchedMetadata, fetchedMetadata...) + + if newLastEvaluatedKey == nil { + assert.Len(t, fetchedMetadata, numBlobs) + break + } else { + assert.Len(t, fetchedMetadata, int(pageSize)) + } + + lastEvaluatedKey = newLastEvaluatedKey + } + + assert.Len(t, allFetchedMetadata, numBlobs) + + // Verify that all blob metadata is fetched and in the correct order + for i, metadata := range allFetchedMetadata { + assert.Equal(t, fmt.Sprintf("blob%d", i), metadata.BlobHash) + assert.Equal(t, fmt.Sprintf("hash%d", i), metadata.MetadataHash) + assert.Equal(t, uint32(i), metadata.ConfirmationInfo.BlobIndex) + } + }) + + // Test invalid batch header hash + t.Run("Fetch All Blobs with Invalid Batch Header Hash", func(t *testing.T) { + invalidBatchHeaderHash := [32]byte{4, 5, 6} + allFetchedMetadata, lastEvaluatedKey, err := sharedStorage.GetAllBlobMetadataByBatchWithPagination(ctx, invalidBatchHeaderHash, 10, nil) + assert.NoError(t, err) + assert.Len(t, allFetchedMetadata, 0) + assert.Nil(t, lastEvaluatedKey) + }) + + // Cleanup: Delete test items + t.Cleanup(func() { + var keys []commondynamodb.Key + for _, blobKey := range blobKeys { + keys = append(keys, commondynamodb.Key{ + "MetadataHash": &types.AttributeValueMemberS{Value: blobKey.MetadataHash}, + "BlobHash": &types.AttributeValueMemberS{Value: blobKey.BlobHash}, + }) + } + deleteItems(t, keys) + }) } func assertMetadata(t *testing.T, blobKey disperser.BlobKey, expectedBlobSize uint, expectedRequestedAt uint64, expectedStatus disperser.BlobStatus, actualMetadata *disperser.BlobMetadata) { diff --git a/disperser/common/inmem/store.go b/disperser/common/inmem/store.go index 8fbab3607b..5142c3f7eb 100644 --- a/disperser/common/inmem/store.go +++ b/disperser/common/inmem/store.go @@ -279,6 +279,48 @@ func (q *BlobStore) GetAllBlobMetadataByBatch(ctx context.Context, batchHeaderHa return metas, nil } +func (q *BlobStore) GetAllBlobMetadataByBatchWithPagination(ctx context.Context, batchHeaderHash [32]byte, limit int32, exclusiveStartKey *disperser.BatchIndexExclusiveStartKey) ([]*disperser.BlobMetadata, *disperser.BatchIndexExclusiveStartKey, error) { + q.mu.RLock() + defer q.mu.RUnlock() + metas := make([]*disperser.BlobMetadata, 0) + foundStart := exclusiveStartKey == nil + + keys := make([]disperser.BlobKey, 0, len(q.Metadata)) + for k, v := range q.Metadata { + if v.ConfirmationInfo != nil && v.ConfirmationInfo.BatchHeaderHash == batchHeaderHash { + keys = append(keys, k) + } + } + sort.Slice(keys, func(i, j int) bool { + return q.Metadata[keys[i]].ConfirmationInfo.BlobIndex < q.Metadata[keys[j]].ConfirmationInfo.BlobIndex + }) + + for _, key := range keys { + meta := q.Metadata[key] + if foundStart { + metas = append(metas, meta) + if len(metas) == int(limit) { + return metas, &disperser.BatchIndexExclusiveStartKey{ + BatchHeaderHash: meta.ConfirmationInfo.BatchHeaderHash[:], + BlobIndex: meta.ConfirmationInfo.BlobIndex, + }, nil + } + } else if exclusiveStartKey != nil && meta.ConfirmationInfo.BlobIndex > uint32(exclusiveStartKey.BlobIndex) { + foundStart = true + metas = append(metas, meta) + if len(metas) == int(limit) { + return metas, &disperser.BatchIndexExclusiveStartKey{ + BatchHeaderHash: meta.ConfirmationInfo.BatchHeaderHash[:], + BlobIndex: meta.ConfirmationInfo.BlobIndex, + }, nil + } + } + } + + // Return all the metas if limit is not reached + return metas, nil, nil +} + func (q *BlobStore) GetBlobMetadata(ctx context.Context, blobKey disperser.BlobKey) (*disperser.BlobMetadata, error) { if meta, ok := q.Metadata[blobKey]; ok { return meta, nil diff --git a/disperser/dataapi/blobs_handlers.go b/disperser/dataapi/blobs_handlers.go index 21d477300d..a77aa28852 100644 --- a/disperser/dataapi/blobs_handlers.go +++ b/disperser/dataapi/blobs_handlers.go @@ -35,6 +35,23 @@ func (s *server) getBlobs(ctx context.Context, limit int) ([]*BlobMetadataRespon return s.convertBlobMetadatasToBlobMetadataResponse(ctx, blobMetadatas) } +func (s *server) getBlobsFromBatchHeaderHash(ctx context.Context, batcherHeaderHash [32]byte, limit int, exclusiveStartKey *disperser.BatchIndexExclusiveStartKey) ([]*BlobMetadataResponse, *disperser.BatchIndexExclusiveStartKey, error) { + blobMetadatas, newExclusiveStartKey, err := s.getBlobMetadataByBatchHeaderHashWithLimit(ctx, batcherHeaderHash, int32(limit), exclusiveStartKey) + if err != nil { + return nil, nil, err + } + if len(blobMetadatas) == 0 { + return nil, nil, errNotFound + } + + responses, err := s.convertBlobMetadatasToBlobMetadataResponse(ctx, blobMetadatas) + if err != nil { + return nil, nil, err + } + + return responses, newExclusiveStartKey, nil +} + func (s *server) convertBlobMetadatasToBlobMetadataResponse(ctx context.Context, metadatas []*disperser.BlobMetadata) ([]*BlobMetadataResponse, error) { var ( err error @@ -96,3 +113,105 @@ func convertMetadataToBlobMetadataResponse(metadata *disperser.BlobMetadata) (*B BlobStatus: metadata.BlobStatus, }, nil } + +func (s *server) getBlobMetadataByBatchesWithLimit(ctx context.Context, limit int) ([]*Batch, []*disperser.BlobMetadata, error) { + var ( + blobMetadatas = make([]*disperser.BlobMetadata, 0) + batches = make([]*Batch, 0) + blobKeyPresence = make(map[string]struct{}) + batchPresence = make(map[string]struct{}) + ) + + for skip := 0; len(blobMetadatas) < limit && skip < limit; skip += maxQueryBatchesLimit { + batchesWithLimit, err := s.subgraphClient.QueryBatchesWithLimit(ctx, maxQueryBatchesLimit, skip) + if err != nil { + s.logger.Error("Failed to query batches", "error", err) + return nil, nil, err + } + + if len(batchesWithLimit) == 0 { + break + } + + for i := range batchesWithLimit { + s.logger.Debug("Getting blob metadata", "batchHeaderHash", batchesWithLimit[i].BatchHeaderHash) + var ( + batch = batchesWithLimit[i] + ) + if batch == nil { + continue + } + batchHeaderHash, err := ConvertHexadecimalToBytes(batch.BatchHeaderHash) + if err != nil { + s.logger.Error("Failed to convert batch header hash to hex string", "error", err) + continue + } + batchKey := string(batchHeaderHash[:]) + if _, found := batchPresence[batchKey]; !found { + batchPresence[batchKey] = struct{}{} + } else { + // The batch has processed, skip it. + s.logger.Error("Getting duplicate batch from the graph", "batch header hash", batchKey) + continue + } + + metadatas, err := s.blobstore.GetAllBlobMetadataByBatch(ctx, batchHeaderHash) + if err != nil { + s.logger.Error("Failed to get blob metadata", "error", err) + continue + } + for _, bm := range metadatas { + blobKey := bm.GetBlobKey().String() + if _, found := blobKeyPresence[blobKey]; !found { + blobKeyPresence[blobKey] = struct{}{} + blobMetadatas = append(blobMetadatas, bm) + } else { + s.logger.Error("Getting duplicate blob key from the blobstore", "blobkey", blobKey) + } + } + batches = append(batches, batch) + if len(blobMetadatas) >= limit { + break + } + } + } + + if len(blobMetadatas) >= limit { + blobMetadatas = blobMetadatas[:limit] + } + + return batches, blobMetadatas, nil +} + +func (s *server) getBlobMetadataByBatchHeaderHashWithLimit(ctx context.Context, batchHeaderHash [32]byte, limit int32, exclusiveStartKey *disperser.BatchIndexExclusiveStartKey) ([]*disperser.BlobMetadata, *disperser.BatchIndexExclusiveStartKey, error) { + var allMetadata []*disperser.BlobMetadata + var nextKey *disperser.BatchIndexExclusiveStartKey = exclusiveStartKey + + const maxLimit int32 = 1000 + remainingLimit := min(limit, maxLimit) + + s.logger.Debug("Getting blob metadata by batch header hash", "batchHeaderHash", batchHeaderHash, "remainingLimit", remainingLimit, "nextKey", nextKey) + for int32(len(allMetadata)) < remainingLimit { + metadatas, newNextKey, err := s.blobstore.GetAllBlobMetadataByBatchWithPagination(ctx, batchHeaderHash, remainingLimit-int32(len(allMetadata)), nextKey) + if err != nil { + s.logger.Error("Failed to get blob metadata", "error", err) + return nil, nil, err + } + + allMetadata = append(allMetadata, metadatas...) + + if newNextKey == nil { + // No more data to fetch + return allMetadata, nil, nil + } + + nextKey = newNextKey + + if int32(len(allMetadata)) == remainingLimit { + // We've reached the limit + break + } + } + + return allMetadata, nextKey, nil +} diff --git a/disperser/dataapi/docs/docs.go b/disperser/dataapi/docs/docs.go index 8e8a2262dc..9beee40527 100644 --- a/disperser/dataapi/docs/docs.go +++ b/disperser/dataapi/docs/docs.go @@ -15,6 +15,64 @@ const docTemplate = `{ "host": "{{.Host}}", "basePath": "{{.BasePath}}", "paths": { + "/feed/batches/{batch_header_hash}/blobs": { + "get": { + "produces": [ + "application/json" + ], + "tags": [ + "Feed" + ], + "summary": "Fetch blob metadata by batch header hash", + "parameters": [ + { + "type": "string", + "description": "Batch Header Hash", + "name": "batch_header_hash", + "in": "path", + "required": true + }, + { + "type": "integer", + "description": "Limit [default: 10]", + "name": "limit", + "in": "query" + }, + { + "type": "string", + "description": "Next page token", + "name": "next_token", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/dataapi.BlobsResponse" + } + }, + "400": { + "description": "error: Bad request", + "schema": { + "$ref": "#/definitions/dataapi.ErrorResponse" + } + }, + "404": { + "description": "error: Not found", + "schema": { + "$ref": "#/definitions/dataapi.ErrorResponse" + } + }, + "500": { + "description": "error: Server error", + "schema": { + "$ref": "#/definitions/dataapi.ErrorResponse" + } + } + } + } + }, "/feed/blobs": { "get": { "produces": [ @@ -656,6 +714,9 @@ const docTemplate = `{ "dataapi.Meta": { "type": "object", "properties": { + "next_token": { + "type": "string" + }, "size": { "type": "integer" } diff --git a/disperser/dataapi/docs/swagger.json b/disperser/dataapi/docs/swagger.json index e19fd43a60..a106eec620 100644 --- a/disperser/dataapi/docs/swagger.json +++ b/disperser/dataapi/docs/swagger.json @@ -11,32 +11,33 @@ "version": "1" }, "paths": { - "/ejector/operators": { - "post": { + "/feed/batches/{batch_header_hash}/blobs": { + "get": { "produces": [ "application/json" ], "tags": [ - "Ejector" + "Feed" ], - "summary": "Eject operators who violate the SLAs during the given time interval", + "summary": "Fetch blob metadata by batch header hash", "parameters": [ { - "type": "integer", - "description": "Lookback window for operator ejection [default: 86400]", - "name": "interval", - "in": "query" + "type": "string", + "description": "Batch Header Hash", + "name": "batch_header_hash", + "in": "path", + "required": true }, { "type": "integer", - "description": "End time for evaluating operator ejection [default: now]", - "name": "end", + "description": "Limit [default: 10]", + "name": "limit", "in": "query" }, { "type": "string", - "description": "Whether it's periodic or urgent ejection request [default: periodic]", - "name": "mode", + "description": "Next page token", + "name": "next_token", "in": "query" } ], @@ -44,7 +45,7 @@ "200": { "description": "OK", "schema": { - "$ref": "#/definitions/dataapi.EjectionResponse" + "$ref": "#/definitions/dataapi.BlobsResponse" } }, "400": { @@ -698,14 +699,6 @@ } } }, - "dataapi.EjectionResponse": { - "type": "object", - "properties": { - "transaction_hash": { - "type": "string" - } - } - }, "dataapi.ErrorResponse": { "type": "object", "properties": { @@ -717,6 +710,9 @@ "dataapi.Meta": { "type": "object", "properties": { + "next_token": { + "type": "string" + }, "size": { "type": "integer" } diff --git a/disperser/dataapi/docs/swagger.yaml b/disperser/dataapi/docs/swagger.yaml index 935033e60c..ccf985794f 100644 --- a/disperser/dataapi/docs/swagger.yaml +++ b/disperser/dataapi/docs/swagger.yaml @@ -66,11 +66,6 @@ definitions: meta: $ref: '#/definitions/dataapi.Meta' type: object - dataapi.EjectionResponse: - properties: - transaction_hash: - type: string - type: object dataapi.ErrorResponse: properties: error: @@ -78,6 +73,8 @@ definitions: type: object dataapi.Meta: properties: + next_token: + type: string size: type: integer type: object @@ -245,21 +242,21 @@ info: title: EigenDA Data Access API version: "1" paths: - /ejector/operators: - post: + /feed/batches/{batch_header_hash}/blobs: + get: parameters: - - description: 'Lookback window for operator ejection [default: 86400]' - in: query - name: interval - type: integer - - description: 'End time for evaluating operator ejection [default: now]' + - description: Batch Header Hash + in: path + name: batch_header_hash + required: true + type: string + - description: 'Limit [default: 10]' in: query - name: end + name: limit type: integer - - description: 'Whether it''s periodic or urgent ejection request [default: - periodic]' + - description: Next page token in: query - name: mode + name: next_token type: string produces: - application/json @@ -267,7 +264,7 @@ paths: "200": description: OK schema: - $ref: '#/definitions/dataapi.EjectionResponse' + $ref: '#/definitions/dataapi.BlobsResponse' "400": description: 'error: Bad request' schema: @@ -280,9 +277,9 @@ paths: description: 'error: Server error' schema: $ref: '#/definitions/dataapi.ErrorResponse' - summary: Eject operators who violate the SLAs during the given time interval + summary: Fetch blob metadata by batch header hash tags: - - Ejector + - Feed /feed/blobs: get: parameters: diff --git a/disperser/dataapi/server.go b/disperser/dataapi/server.go index 6f09293a47..b0c327ef6a 100644 --- a/disperser/dataapi/server.go +++ b/disperser/dataapi/server.go @@ -2,6 +2,8 @@ package dataapi import ( "context" + "encoding/base64" + "encoding/json" "errors" "fmt" "math/big" @@ -43,7 +45,7 @@ const ( maxThroughputAge = 10 maxMetricAage = 10 maxFeedBlobsAge = 10 - maxFeedBlobAage = 300 // this is completely static + maxFeedBlobAge = 300 // this is completely static maxDisperserAvailabilityAge = 3 maxChurnerAvailabilityAge = 3 maxBatcherAvailabilityAge = 3 @@ -93,7 +95,8 @@ type ( } Meta struct { - Size int `json:"size"` + Size int `json:"size"` + NextToken string `json:"next_token,omitempty"` } BlobsResponse struct { @@ -194,7 +197,6 @@ func NewServer( } if eigenDAGRPCServiceChecker == nil { - eigenDAGRPCServiceChecker = NewEigenDAServiceHealthCheck(grpcConn, config.DisperserHostname, config.ChurnerHostname) } @@ -231,13 +233,13 @@ func (s *server) Start() error { basePath := "/api/v1" docs.SwaggerInfo.BasePath = basePath docs.SwaggerInfo.Host = os.Getenv("SWAGGER_HOST") - v1 := router.Group(basePath) { feed := v1.Group("/feed") { feed.GET("/blobs", s.FetchBlobsHandler) feed.GET("/blobs/:blob_key", s.FetchBlobHandler) + feed.GET("/batches/:batch_header_hash/blobs", s.FetchBlobsFromBatchHeaderHash) } operatorsInfo := v1.Group("/operators-info") { @@ -333,10 +335,118 @@ func (s *server) FetchBlobHandler(c *gin.Context) { } s.metrics.IncrementSuccessfulRequestNum("FetchBlob") - c.Writer.Header().Set(cacheControlParam, fmt.Sprintf("max-age=%d", maxFeedBlobAage)) + c.Writer.Header().Set(cacheControlParam, fmt.Sprintf("max-age=%d", maxFeedBlobAge)) c.JSON(http.StatusOK, metadata) } +// FetchBlobsFromBatchHeaderHash godoc +// +// @Summary Fetch blob metadata by batch header hash +// @Tags Feed +// @Produce json +// @Param batch_header_hash path string true "Batch Header Hash" +// @Param limit query int false "Limit [default: 10]" +// @Param next_token query string false "Next page token" +// @Success 200 {object} BlobsResponse +// @Failure 400 {object} ErrorResponse "error: Bad request" +// @Failure 404 {object} ErrorResponse "error: Not found" +// @Failure 500 {object} ErrorResponse "error: Server error" +// @Router /feed/batches/{batch_header_hash}/blobs [get] +func (s *server) FetchBlobsFromBatchHeaderHash(c *gin.Context) { + timer := prometheus.NewTimer(prometheus.ObserverFunc(func(f float64) { + s.metrics.ObserveLatency("FetchBlobsFromBatchHeaderHash", f*1000) // make milliseconds + })) + defer timer.ObserveDuration() + + batchHeaderHash := c.Param("batch_header_hash") + batchHeaderHashBytes, err := ConvertHexadecimalToBytes([]byte(batchHeaderHash)) + if err != nil { + s.metrics.IncrementFailedRequestNum("FetchBlobsFromBatchHeaderHash") + errorResponse(c, fmt.Errorf("invalid batch header hash")) + return + } + + limit, err := strconv.Atoi(c.DefaultQuery("limit", "10")) + if err != nil { + s.metrics.IncrementFailedRequestNum("FetchBlobsFromBatchHeaderHash") + errorResponse(c, fmt.Errorf("invalid limit parameter")) + return + } + if limit <= 0 || limit > 1000 { + s.metrics.IncrementFailedRequestNum("FetchBlobsFromBatchHeaderHash") + errorResponse(c, fmt.Errorf("limit must be between 0 and 1000")) + return + } + + var exclusiveStartKey *disperser.BatchIndexExclusiveStartKey + nextToken := c.Query("next_token") + if nextToken != "" { + exclusiveStartKey, err = decodeNextToken(nextToken) + if err != nil { + s.metrics.IncrementFailedRequestNum("FetchBlobsFromBatchHeaderHash") + errorResponse(c, fmt.Errorf("invalid next_token")) + return + } + } + + metadatas, newExclusiveStartKey, err := s.getBlobsFromBatchHeaderHash(c.Request.Context(), batchHeaderHashBytes, limit, exclusiveStartKey) + if err != nil { + s.metrics.IncrementFailedRequestNum("FetchBlobsFromBatchHeaderHash") + errorResponse(c, err) + return + } + + var nextPageToken string + if newExclusiveStartKey != nil { + nextPageToken, err = encodeNextToken(newExclusiveStartKey) + if err != nil { + s.metrics.IncrementFailedRequestNum("FetchBlobsFromBatchHeaderHash") + errorResponse(c, fmt.Errorf("failed to generate next page token")) + return + } + } + + s.metrics.IncrementSuccessfulRequestNum("FetchBlobsFromBatchHeaderHash") + c.Writer.Header().Set(cacheControlParam, fmt.Sprintf("max-age=%d", maxFeedBlobAge)) + c.JSON(http.StatusOK, BlobsResponse{ + Meta: Meta{ + Size: len(metadatas), + NextToken: nextPageToken, + }, + Data: metadatas, + }) +} + +func decodeNextToken(token string) (*disperser.BatchIndexExclusiveStartKey, error) { + // Decode the base64 string + decodedBytes, err := base64.URLEncoding.DecodeString(token) + if err != nil { + return nil, fmt.Errorf("failed to decode token: %w", err) + } + + // Unmarshal the JSON into a BatchIndexExclusiveStartKey + var key disperser.BatchIndexExclusiveStartKey + err = json.Unmarshal(decodedBytes, &key) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal token: %w", err) + } + + return &key, nil +} + +func encodeNextToken(key *disperser.BatchIndexExclusiveStartKey) (string, error) { + // Marshal the key to JSON + jsonBytes, err := json.Marshal(key) + if err != nil { + return "", fmt.Errorf("failed to marshal key: %w", err) + } + + // Encode the JSON as a base64 string + token := base64.URLEncoding.EncodeToString(jsonBytes) + + return token, nil +} + // FetchBlobsHandler godoc // // @Summary Fetch blobs metadata list @@ -356,7 +466,14 @@ func (s *server) FetchBlobsHandler(c *gin.Context) { limit, err := strconv.Atoi(c.DefaultQuery("limit", "10")) if err != nil { - limit = 10 + s.metrics.IncrementFailedRequestNum("FetchBlobsFromBatchHeaderHash") + errorResponse(c, fmt.Errorf("invalid limit parameter")) + return + } + if limit <= 0 { + s.metrics.IncrementFailedRequestNum("FetchBlobsFromBatchHeaderHash") + errorResponse(c, fmt.Errorf("limit must be greater than 0")) + return } metadatas, err := s.getBlobs(c.Request.Context(), limit) @@ -848,75 +965,6 @@ func (s *server) FetchBatcherAvailability(c *gin.Context) { }) } -func (s *server) getBlobMetadataByBatchesWithLimit(ctx context.Context, limit int) ([]*Batch, []*disperser.BlobMetadata, error) { - var ( - blobMetadatas = make([]*disperser.BlobMetadata, 0) - batches = make([]*Batch, 0) - blobKeyPresence = make(map[string]struct{}) - batchPresence = make(map[string]struct{}) - ) - - for skip := 0; len(blobMetadatas) < limit && skip < limit; skip += maxQueryBatchesLimit { - batchesWithLimit, err := s.subgraphClient.QueryBatchesWithLimit(ctx, maxQueryBatchesLimit, skip) - if err != nil { - s.logger.Error("Failed to query batches", "error", err) - return nil, nil, err - } - - if len(batchesWithLimit) == 0 { - break - } - - for i := range batchesWithLimit { - s.logger.Debug("Getting blob metadata", "batchHeaderHash", batchesWithLimit[i].BatchHeaderHash) - var ( - batch = batchesWithLimit[i] - ) - if batch == nil { - continue - } - batchHeaderHash, err := ConvertHexadecimalToBytes(batch.BatchHeaderHash) - if err != nil { - s.logger.Error("Failed to convert batch header hash to hex string", "error", err) - continue - } - batchKey := string(batchHeaderHash[:]) - if _, found := batchPresence[batchKey]; !found { - batchPresence[batchKey] = struct{}{} - } else { - // The batch has processed, skip it. - s.logger.Error("Getting duplicate batch from the graph", "batch header hash", batchKey) - continue - } - - metadatas, err := s.blobstore.GetAllBlobMetadataByBatch(ctx, batchHeaderHash) - if err != nil { - s.logger.Error("Failed to get blob metadata", "error", err) - continue - } - for _, bm := range metadatas { - blobKey := bm.GetBlobKey().String() - if _, found := blobKeyPresence[blobKey]; !found { - blobKeyPresence[blobKey] = struct{}{} - blobMetadatas = append(blobMetadatas, bm) - } else { - s.logger.Error("Getting duplicate blob key from the blobstore", "blobkey", blobKey) - } - } - batches = append(batches, batch) - if len(blobMetadatas) >= limit { - break - } - } - } - - if len(blobMetadatas) >= limit { - blobMetadatas = blobMetadatas[:limit] - } - - return batches, blobMetadatas, nil -} - func errorResponse(c *gin.Context, err error) { _ = c.Error(err) var code int diff --git a/disperser/dataapi/server_test.go b/disperser/dataapi/server_test.go index 6e600bb3c7..db39168f5b 100644 --- a/disperser/dataapi/server_test.go +++ b/disperser/dataapi/server_test.go @@ -68,8 +68,6 @@ var ( }, }) testDataApiServer = dataapi.NewServer(config, blobstore, prometheusClient, subgraphClient, mockTx, mockChainState, mockLogger, dataapi.NewMetrics(nil, "9001", mockLogger), &MockGRPCConnection{}, nil, nil) - expectedBatchHeaderHash = [32]byte{1, 2, 3} - expectedBlobIndex = uint32(1) expectedRequestedAt = uint64(5567830000000000000) expectedDataLength = 32 expectedBatchId = uint32(99) @@ -153,7 +151,9 @@ func TestFetchBlobHandler(t *testing.T) { blob := makeTestBlob(0, 80) key := queueBlob(t, &blob, blobstore) - markBlobConfirmed(t, &blob, key, expectedBatchHeaderHash, blobstore) + expectedBatchHeaderHash := [32]byte{1, 2, 3} + expectedBlobIndex := uint32(1) + markBlobConfirmed(t, &blob, key, expectedBlobIndex, expectedBatchHeaderHash, blobstore) blobKey := key.String() r.GET("/v1/feed/blobs/:blob_key", testDataApiServer.FetchBlobHandler) @@ -202,7 +202,7 @@ func TestFetchBlobsHandler(t *testing.T) { batchHeaderHashBytes := []byte(batch.BatchHeaderHash) batchHeaderHash, err := dataapi.ConvertHexadecimalToBytes(batchHeaderHashBytes) assert.NoError(t, err) - markBlobConfirmed(t, &blob, key, batchHeaderHash, blobstore) + markBlobConfirmed(t, &blob, key, 1, batchHeaderHash, blobstore) } mockSubgraphApi.On("QueryBatches").Return(subgraphBatches, nil) @@ -229,6 +229,118 @@ func TestFetchBlobsHandler(t *testing.T) { assert.Equal(t, 2, len(response.Data)) } +func TestFetchBlobsFromBatchHeaderHash(t *testing.T) { + r := setUpRouter() + + batchHeaderHash := "6E2EFA6EB7AE40CE7A65B465679DE5649F994296D18C075CF2C490564BBF7CA5" + batchHeaderHashBytes, err := dataapi.ConvertHexadecimalToBytes([]byte(batchHeaderHash)) + assert.NoError(t, err) + + blob1 := makeTestBlob(0, 80) + key1 := queueBlob(t, &blob1, blobstore) + + blob2 := makeTestBlob(0, 80) + key2 := queueBlob(t, &blob2, blobstore) + + markBlobConfirmed(t, &blob1, key1, 1, batchHeaderHashBytes, blobstore) + markBlobConfirmed(t, &blob2, key2, 2, batchHeaderHashBytes, blobstore) + + r.GET("/v1/feed/batches/:batch_header_hash/blobs", testDataApiServer.FetchBlobsFromBatchHeaderHash) + + w := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/v1/feed/batches/"+batchHeaderHash+"/blobs?limit=1", nil) + r.ServeHTTP(w, req) + + res := w.Result() + defer res.Body.Close() + + data, err := io.ReadAll(res.Body) + assert.NoError(t, err) + + var response dataapi.BlobsResponse + err = json.Unmarshal(data, &response) + assert.NoError(t, err) + assert.NotNil(t, response) + + assert.Equal(t, http.StatusOK, res.StatusCode) + assert.Equal(t, 1, response.Meta.Size) + assert.Equal(t, hex.EncodeToString(batchHeaderHashBytes[:]), response.Data[0].BatchHeaderHash) + assert.Equal(t, uint32(1), uint32(response.Data[0].BlobIndex)) + + // With the next_token query parameter set, the response should contain the next token + w = httptest.NewRecorder() + req = httptest.NewRequest(http.MethodGet, "/v1/feed/batches/"+batchHeaderHash+"/blobs?limit=1&next_token="+response.Meta.NextToken, nil) + r.ServeHTTP(w, req) + + res = w.Result() + defer res.Body.Close() + + data, err = io.ReadAll(res.Body) + assert.NoError(t, err) + + err = json.Unmarshal(data, &response) + assert.NoError(t, err) + assert.NotNil(t, response) + + assert.Equal(t, http.StatusOK, res.StatusCode) + assert.Equal(t, 1, response.Meta.Size) + assert.Equal(t, hex.EncodeToString(batchHeaderHashBytes[:]), response.Data[0].BatchHeaderHash) + assert.Equal(t, uint32(2), uint32(response.Data[0].BlobIndex)) + + // With the next_token query parameter set to an invalid value, the response should contain an error + w = httptest.NewRecorder() + req = httptest.NewRequest(http.MethodGet, "/v1/feed/batches/"+batchHeaderHash+"/blobs?limit=1&next_token=invalid", nil) + r.ServeHTTP(w, req) + + res = w.Result() + defer res.Body.Close() + + data, err = io.ReadAll(res.Body) + assert.NoError(t, err) + + var errorResponse dataapi.ErrorResponse + err = json.Unmarshal(data, &errorResponse) + assert.NoError(t, err) + + assert.Equal(t, http.StatusInternalServerError, res.StatusCode) + assert.Equal(t, "invalid next_token", errorResponse.Error) + + // Fetch both blobs when no limit is set + w = httptest.NewRecorder() + req = httptest.NewRequest(http.MethodGet, "/v1/feed/batches/"+batchHeaderHash+"/blobs", nil) + r.ServeHTTP(w, req) + + res = w.Result() + defer res.Body.Close() + + data, err = io.ReadAll(res.Body) + assert.NoError(t, err) + + err = json.Unmarshal(data, &response) + assert.NoError(t, err) + assert.NotNil(t, response) + + assert.Equal(t, http.StatusOK, res.StatusCode) + assert.Equal(t, 2, response.Meta.Size) + + // When the batch header hash is invalid, the response should contain an error + w = httptest.NewRecorder() + req = httptest.NewRequest(http.MethodGet, "/v1/feed/batches/invalid/blobs", nil) + r.ServeHTTP(w, req) + + res = w.Result() + defer res.Body.Close() + + data, err = io.ReadAll(res.Body) + assert.NoError(t, err) + + err = json.Unmarshal(data, &errorResponse) + assert.NoError(t, err) + + assert.Equal(t, http.StatusInternalServerError, res.StatusCode) + assert.Equal(t, "invalid batch header hash", errorResponse.Error) +} + func TestFetchMetricsHandler(t *testing.T) { defer goleak.VerifyNone(t) @@ -244,7 +356,7 @@ func TestFetchMetricsHandler(t *testing.T) { batchHeaderHash, err := dataapi.ConvertHexadecimalToBytes(batchHeaderHashBytes) assert.NoError(t, err) - markBlobConfirmed(t, &blob, key, batchHeaderHash, blobstore) + markBlobConfirmed(t, &blob, key, 1, batchHeaderHash, blobstore) } s := new(model.SampleStream) @@ -1423,7 +1535,7 @@ func queueBlob(t *testing.T, blob *core.Blob, queue disperser.BlobStore) dispers return key } -func markBlobConfirmed(t *testing.T, blob *core.Blob, key disperser.BlobKey, batchHeaderHash [32]byte, queue disperser.BlobStore) { +func markBlobConfirmed(t *testing.T, blob *core.Blob, key disperser.BlobKey, blobIndex uint32, batchHeaderHash [32]byte, queue disperser.BlobStore) { // simulate blob confirmation var commitX, commitY fp.Element _, err := commitX.SetString("21661178944771197726808973281966770251114553549453983978976194544185382599016") @@ -1437,7 +1549,7 @@ func markBlobConfirmed(t *testing.T, blob *core.Blob, key disperser.BlobKey, bat confirmationInfo := &disperser.ConfirmationInfo{ BatchHeaderHash: batchHeaderHash, - BlobIndex: expectedBlobIndex, + BlobIndex: blobIndex, SignatoryRecordHash: expectedSignatoryRecordHash, ReferenceBlockNumber: expectedReferenceBlockNumber, BatchRoot: expectedBatchRoot, diff --git a/disperser/dataapi/utils.go b/disperser/dataapi/utils.go index 148504cf51..c2548b96d4 100644 --- a/disperser/dataapi/utils.go +++ b/disperser/dataapi/utils.go @@ -23,7 +23,7 @@ func ConvertHexadecimalToBytes(byteHash []byte) ([32]byte, error) { // We expect the resulting byte slice to have a length of 32 bytes. if len(decodedBytes) != 32 { - return [32]byte{}, errors.New("error decoding hash") + return [32]byte{}, errors.New("error decoding hash, invalid length") } // Convert the byte slice to a [32]byte array diff --git a/disperser/disperser.go b/disperser/disperser.go index 8603b4abfa..cae2980df3 100644 --- a/disperser/disperser.go +++ b/disperser/disperser.go @@ -135,6 +135,13 @@ type BlobStoreExclusiveStartKey struct { RequestedAt int64 // RequestedAt is epoch time in seconds } +type BatchIndexExclusiveStartKey struct { + BlobHash BlobHash + MetadataHash MetadataHash + BatchHeaderHash []byte + BlobIndex uint32 +} + type BlobStore interface { // StoreBlob adds a blob to the queue and returns a key that can be used to retrieve the blob later StoreBlob(ctx context.Context, blob *core.Blob, requestedAt uint64) (BlobKey, error) @@ -169,6 +176,8 @@ type BlobStore interface { GetBlobMetadataByStatusWithPagination(ctx context.Context, blobStatus BlobStatus, limit int32, exclusiveStartKey *BlobStoreExclusiveStartKey) ([]*BlobMetadata, *BlobStoreExclusiveStartKey, error) // GetAllBlobMetadataByBatch returns the metadata of all the blobs in the batch. GetAllBlobMetadataByBatch(ctx context.Context, batchHeaderHash [32]byte) ([]*BlobMetadata, error) + // GetAllBlobMetadataByBatchWithPagination returns all the blobs in the batch using pagination + GetAllBlobMetadataByBatchWithPagination(ctx context.Context, batchHeaderHash [32]byte, limit int32, exclusiveStartKey *BatchIndexExclusiveStartKey) ([]*BlobMetadata, *BatchIndexExclusiveStartKey, error) // GetBlobMetadata returns a blob metadata given a metadata key GetBlobMetadata(ctx context.Context, blobKey BlobKey) (*BlobMetadata, error) // HandleBlobFailure handles a blob failure by either incrementing the retry count or marking the blob as failed From b53aee8f1aae6af1fab44c66f30c374aad7ed798 Mon Sep 17 00:00:00 2001 From: Oksana <107276324+Ocheretovich@users.noreply.github.com> Date: Wed, 14 Aug 2024 08:42:10 +0300 Subject: [PATCH 2/5] Update README.md (#694) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6556661757..b976093c91 100644 --- a/README.md +++ b/README.md @@ -38,4 +38,4 @@ We welcome all contributions! There are many ways to contribute to the project, - [Open an Issue](https://github.com/Layr-Labs/eigenda/issues/new/choose) - [EigenLayer/EigenDA forum](https://forum.eigenlayer.xyz/c/eigenda/9) - [Email](mailto:eigenda-support@eigenlabs.org) -- [Follow us on Twitter](https://twitter.com/eigen_da) +- [Follow us on X](https://x.com/eigen_da) From 12f1a6f6c8951cda36b93f1060e8ce441b06b6ec Mon Sep 17 00:00:00 2001 From: Ian Shim <100327837+ian-shim@users.noreply.github.com> Date: Thu, 15 Aug 2024 09:52:10 -0700 Subject: [PATCH 3/5] [node] `AttestBatch` endpoint (#676) --- api/clients/node_client.go | 20 +-- core/validator.go | 17 +- go.mod | 2 +- node/grpc/server.go | 112 ++++++++++++- node/grpc/server_test.go | 46 +++++- node/grpc/utils.go | 298 ----------------------------------- node/node.go | 59 ++++++- node/store.go | 197 +++++++++++++++++------ node/store_test.go | 74 ++++++++- node/store_utils.go | 197 +++++++++++++++++++++++ node/store_utils_test.go | 83 ++++++++++ node/utils.go | 314 +++++++++++++++++++++++-------------- node/utils_test.go | 26 --- test/integration_test.go | 2 +- 14 files changed, 921 insertions(+), 526 deletions(-) delete mode 100644 node/grpc/utils.go create mode 100644 node/store_utils.go create mode 100644 node/store_utils_test.go delete mode 100644 node/utils_test.go diff --git a/api/clients/node_client.go b/api/clients/node_client.go index 56dd5912b1..9d620da2ae 100644 --- a/api/clients/node_client.go +++ b/api/clients/node_client.go @@ -5,10 +5,10 @@ import ( "errors" "time" - "github.com/Layr-Labs/eigenda/api/grpc/node" + grpcnode "github.com/Layr-Labs/eigenda/api/grpc/node" "github.com/Layr-Labs/eigenda/core" "github.com/Layr-Labs/eigenda/encoding" - node_utils "github.com/Layr-Labs/eigenda/node/grpc" + "github.com/Layr-Labs/eigenda/node" "github.com/wealdtech/go-merkletree/v2" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" @@ -50,11 +50,11 @@ func (c client) GetBlobHeader( } defer conn.Close() - n := node.NewRetrievalClient(conn) + n := grpcnode.NewRetrievalClient(conn) nodeCtx, cancel := context.WithTimeout(ctx, c.timeout) defer cancel() - request := &node.GetBlobHeaderRequest{ + request := &grpcnode.GetBlobHeaderRequest{ BatchHeaderHash: batchHeaderHash[:], BlobIndex: blobIndex, } @@ -64,7 +64,7 @@ func (c client) GetBlobHeader( return nil, nil, err } - blobHeader, err := node_utils.GetBlobHeaderFromProto(reply.GetBlobHeader()) + blobHeader, err := node.GetBlobHeaderFromProto(reply.GetBlobHeader()) if err != nil { return nil, nil, err } @@ -99,11 +99,11 @@ func (c client) GetChunks( return } - n := node.NewRetrievalClient(conn) + n := grpcnode.NewRetrievalClient(conn) nodeCtx, cancel := context.WithTimeout(ctx, c.timeout) defer cancel() - request := &node.RetrieveChunksRequest{ + request := &grpcnode.RetrieveChunksRequest{ BatchHeaderHash: batchHeaderHash[:], BlobIndex: blobIndex, QuorumId: uint32(quorumID), @@ -123,11 +123,11 @@ func (c client) GetChunks( for i, data := range reply.GetChunks() { var chunk *encoding.Frame switch reply.GetChunkEncodingFormat() { - case node.ChunkEncodingFormat_GNARK: + case grpcnode.ChunkEncodingFormat_GNARK: chunk, err = new(encoding.Frame).DeserializeGnark(data) - case node.ChunkEncodingFormat_GOB: + case grpcnode.ChunkEncodingFormat_GOB: chunk, err = new(encoding.Frame).Deserialize(data) - case node.ChunkEncodingFormat_UNKNOWN: + case grpcnode.ChunkEncodingFormat_UNKNOWN: // For backward compatibility, we fallback the UNKNOWN to GOB chunk, err = new(encoding.Frame).Deserialize(data) if err != nil { diff --git a/core/validator.go b/core/validator.go index 9309cad0f3..f4ca9c9027 100644 --- a/core/validator.go +++ b/core/validator.go @@ -88,7 +88,11 @@ func (v *shardValidator) UpdateOperatorID(operatorID OperatorID) { } func (v *shardValidator) ValidateBatch(batchHeader *BatchHeader, blobs []*BlobMessage, operatorState *OperatorState, pool common.WorkerPool) error { - err := validateBatchHeaderRoot(batchHeader, blobs) + headers := make([]*BlobHeader, len(blobs)) + for i, blob := range blobs { + headers[i] = blob.BlobHeader + } + err := ValidateBatchHeaderRoot(batchHeader, headers) if err != nil { return err } @@ -207,18 +211,11 @@ func (v *shardValidator) VerifyBlobLengthWorker(blobCommitments encoding.BlobCom out <- nil } -func validateBatchHeaderRoot(batchHeader *BatchHeader, blobs []*BlobMessage) error { +func ValidateBatchHeaderRoot(batchHeader *BatchHeader, blobHeaders []*BlobHeader) error { // Check the batch header root - - headers := make([]*BlobHeader, len(blobs)) - - for i, blob := range blobs { - headers[i] = blob.BlobHeader - } - derivedHeader := &BatchHeader{} - _, err := derivedHeader.SetBatchRoot(headers) + _, err := derivedHeader.SetBatchRoot(blobHeaders) if err != nil { return fmt.Errorf("failed to compute batch header root: %w", err) } diff --git a/go.mod b/go.mod index 6ecc3834f7..429029d592 100644 --- a/go.mod +++ b/go.mod @@ -15,6 +15,7 @@ require ( github.com/fxamacker/cbor/v2 v2.5.0 github.com/gin-contrib/logger v0.2.6 github.com/gin-gonic/gin v1.9.1 + github.com/golang/protobuf v1.5.4 github.com/hashicorp/go-multierror v1.1.1 github.com/joho/godotenv v1.5.1 github.com/onsi/ginkgo/v2 v2.11.0 @@ -99,7 +100,6 @@ require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt v3.2.2+incompatible // indirect github.com/golang-jwt/jwt/v4 v4.5.0 // indirect - github.com/golang/protobuf v1.5.4 // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect github.com/hashicorp/errwrap v1.0.0 // indirect github.com/hashicorp/go-bexpr v0.1.10 // indirect diff --git a/node/grpc/server.go b/node/grpc/server.go index 7329e638d7..65ee30df61 100644 --- a/node/grpc/server.go +++ b/node/grpc/server.go @@ -5,6 +5,7 @@ import ( "encoding/hex" "errors" "fmt" + "reflect" "runtime" "sync" "time" @@ -21,6 +22,8 @@ import ( "github.com/Layr-Labs/eigensdk-go/logging" "github.com/golang/protobuf/ptypes/wrappers" "github.com/shirou/gopsutil/mem" + "github.com/wealdtech/go-merkletree/v2" + "github.com/wealdtech/go-merkletree/v2/keccak256" _ "go.uber.org/automaxprocs" @@ -155,12 +158,12 @@ func (s *Server) handleStoreChunksRequest(ctx context.Context, in *pb.StoreChunk start := time.Now() // Get batch header hash - batchHeader, err := GetBatchHeader(in) + batchHeader, err := node.GetBatchHeader(in.GetBatchHeader()) if err != nil { return nil, err } - blobs, err := GetBlobMessages(in.GetBlobs(), s.node.Config.NumBatchDeserializationWorkers) + blobs, err := node.GetBlobMessages(in.GetBlobs(), s.node.Config.NumBatchDeserializationWorkers) if err != nil { return nil, err } @@ -196,7 +199,7 @@ func (s *Server) validateStoreChunkRequest(in *pb.StoreChunksRequest) error { if blob.GetHeader() == nil { return api.NewInvalidArgError("missing blob header in request") } - if ValidatePointsFromBlobHeader(blob.GetHeader()) != nil { + if node.ValidatePointsFromBlobHeader(blob.GetHeader()) != nil { return api.NewInvalidArgError("invalid points contained in the blob header in request") } if len(blob.GetHeader().GetQuorumHeaders()) == 0 { @@ -264,7 +267,7 @@ func (s *Server) validateStoreBlobsRequest(in *pb.StoreBlobsRequest) error { if blob.GetHeader() == nil { return api.NewInvalidArgError("missing blob header in request") } - if ValidatePointsFromBlobHeader(blob.GetHeader()) != nil { + if node.ValidatePointsFromBlobHeader(blob.GetHeader()) != nil { return api.NewInvalidArgError("invalid points contained in the blob header in request") } if len(blob.GetHeader().GetQuorumHeaders()) == 0 { @@ -308,7 +311,7 @@ func (s *Server) StoreBlobs(ctx context.Context, in *pb.StoreBlobsRequest) (*pb. s.node.Logger.Info("StoreBlobs RPC request received", "numBlobs", len(in.Blobs), "reqMsgSize", proto.Size(in), "blobHeadersSize", blobHeadersSize, "bundleSize", bundleSize, "referenceBlockNumber", in.GetReferenceBlockNumber()) // Process the request - blobs, err := GetBlobMessages(in.GetBlobs(), s.node.Config.NumBatchDeserializationWorkers) + blobs, err := node.GetBlobMessages(in.GetBlobs(), s.node.Config.NumBatchDeserializationWorkers) if err != nil { return nil, err } @@ -334,7 +337,44 @@ func (s *Server) StoreBlobs(ctx context.Context, in *pb.StoreBlobsRequest) (*pb. } func (s *Server) AttestBatch(ctx context.Context, in *pb.AttestBatchRequest) (*pb.AttestBatchReply, error) { - return nil, errors.New("AttestBatch is not implemented") + start := time.Now() + + // Validate the batch root + blobHeaderHashes := make([][32]byte, len(in.GetBlobHeaderHashes())) + for i, hash := range in.GetBlobHeaderHashes() { + if len(hash) != 32 { + return nil, api.NewInvalidArgError("invalid blob header hash") + } + var h [32]byte + copy(h[:], hash) + blobHeaderHashes[i] = h + } + batchHeader, err := node.GetBatchHeader(in.GetBatchHeader()) + if err != nil { + return nil, fmt.Errorf("failed to get the batch header: %w", err) + } + err = s.node.ValidateBatchContents(ctx, blobHeaderHashes, batchHeader) + if err != nil { + return nil, fmt.Errorf("failed to validate the batch header root: %w", err) + } + + // Store the mapping from batch header + blob index to blob header hashes + err = s.node.Store.StoreBatchBlobMapping(ctx, batchHeader, blobHeaderHashes) + if err != nil { + return nil, fmt.Errorf("failed to store the batch blob mapping: %w", err) + } + + // Sign the batch header + batchHeaderHash, err := batchHeader.GetBatchHeaderHash() + if err != nil { + return nil, fmt.Errorf("failed to get the batch header hash: %w", err) + } + sig := s.node.KeyPair.SignMessage(batchHeaderHash) + + s.node.Logger.Info("AttestBatch complete", "duration", time.Since(start)) + return &pb.AttestBatchReply{ + Signature: sig.Serialize(), + }, nil } func (s *Server) RetrieveChunks(ctx context.Context, in *pb.RetrieveChunksRequest) (*pb.RetrieveChunksReply, error) { @@ -445,6 +485,64 @@ func (s *Server) GetBlobHeader(ctx context.Context, in *pb.GetBlobHeaderRequest) }, nil } +// rebuildMerkleTree rebuilds the merkle tree from the blob headers and batch header. +func (s *Server) rebuildMerkleTree(batchHeaderHash [32]byte) (*merkletree.MerkleTree, error) { + batchHeaderBytes, err := s.node.Store.GetBatchHeader(context.Background(), batchHeaderHash) + if err != nil { + return nil, errors.New("failed to get the batch header from Store") + } + + batchHeader, err := new(core.BatchHeader).Deserialize(batchHeaderBytes) + if err != nil { + return nil, err + } + + blobIndex := 0 + leafs := make([][]byte, 0) + for { + blobHeaderBytes, err := s.node.Store.GetBlobHeader(context.Background(), batchHeaderHash, blobIndex) + if err != nil { + if errors.Is(err, node.ErrKeyNotFound) { + break + } + return nil, err + } + + var protoBlobHeader pb.BlobHeader + err = proto.Unmarshal(blobHeaderBytes, &protoBlobHeader) + if err != nil { + return nil, err + } + + blobHeader, err := node.GetBlobHeaderFromProto(&protoBlobHeader) + if err != nil { + return nil, err + } + + blobHeaderHash, err := blobHeader.GetBlobHeaderHash() + if err != nil { + return nil, err + } + leafs = append(leafs, blobHeaderHash[:]) + blobIndex++ + } + + if len(leafs) == 0 { + return nil, errors.New("no blob header found") + } + + tree, err := merkletree.NewTree(merkletree.WithData(leafs), merkletree.WithHashType(keccak256.New())) + if err != nil { + return nil, err + } + + if !reflect.DeepEqual(tree.Root(), batchHeader.BatchRoot[:]) { + return nil, errors.New("invalid batch header") + } + + return tree, nil +} + func (s *Server) getBlobHeader(ctx context.Context, batchHeaderHash [32]byte, blobIndex int) (*core.BlobHeader, *pb.BlobHeader, error) { blobHeaderBytes, err := s.node.Store.GetBlobHeader(ctx, batchHeaderHash, blobIndex) @@ -458,7 +556,7 @@ func (s *Server) getBlobHeader(ctx context.Context, batchHeaderHash [32]byte, bl return nil, nil, err } - blobHeader, err := GetBlobHeaderFromProto(&protoBlobHeader) + blobHeader, err := node.GetBlobHeaderFromProto(&protoBlobHeader) if err != nil { return nil, nil, err } diff --git a/node/grpc/server_test.go b/node/grpc/server_test.go index ca3624f0d0..714fb5057a 100644 --- a/node/grpc/server_test.go +++ b/node/grpc/server_test.go @@ -416,16 +416,46 @@ func TestStoreBlobs(t *testing.T) { func TestAttestBatch(t *testing.T) { server := newTestServer(t, true) - reqToCopy, _, _, _, _ := makeStoreChunksRequest(t, 66, 33) + reqToCopy, _, _, blobHeaders, _ := makeStoreChunksRequest(t, 66, 33) reqToCopy.BatchHeader = nil - req := &pb.AttestBatchRequest{ - BatchHeader: reqToCopy.BatchHeader, - BlobHeaderHashes: [][]byte{}, + req := &pb.StoreBlobsRequest{ + Blobs: reqToCopy.Blobs, + ReferenceBlockNumber: 1, } - reply, err := server.AttestBatch(context.Background(), req) - assert.Nil(t, reply) - assert.Error(t, err) - assert.Equal(t, strings.Compare(err.Error(), "AttestBatch is not implemented"), 0) + reply, err := server.StoreBlobs(context.Background(), req) + assert.NoError(t, err) + assert.NotNil(t, reply.GetSignatures()) + + assert.Len(t, blobHeaders, 2) + bhh0, err := blobHeaders[0].GetBlobHeaderHash() + assert.NoError(t, err) + bhh1, err := blobHeaders[1].GetBlobHeaderHash() + assert.NoError(t, err) + batchHeader := &core.BatchHeader{ + ReferenceBlockNumber: 1, + BatchRoot: [32]byte{}, + } + _, err = batchHeader.SetBatchRoot([]*core.BlobHeader{blobHeaders[0], blobHeaders[1]}) + assert.NoError(t, err) + attestReq := &pb.AttestBatchRequest{ + BatchHeader: &pb.BatchHeader{ + BatchRoot: batchHeader.BatchRoot[:], + ReferenceBlockNumber: 1, + }, + BlobHeaderHashes: [][]byte{bhh0[:], bhh1[:]}, + } + attestReply, err := server.AttestBatch(context.Background(), attestReq) + assert.NotNil(t, reply) + assert.NoError(t, err) + sig := attestReply.GetSignature() + assert.NotNil(t, sig) + batchHeaderHash, err := batchHeader.GetBatchHeaderHash() + assert.NoError(t, err) + point, err := new(core.Signature).Deserialize(sig) + assert.NoError(t, err) + s := &core.Signature{G1Point: point} + ok := s.Verify(keyPair.GetPubKeyG2(), batchHeaderHash) + assert.True(t, ok) } func TestRetrieveChunks(t *testing.T) { diff --git a/node/grpc/utils.go b/node/grpc/utils.go deleted file mode 100644 index cbdd4735bf..0000000000 --- a/node/grpc/utils.go +++ /dev/null @@ -1,298 +0,0 @@ -package grpc - -import ( - "context" - "errors" - "fmt" - "reflect" - - "github.com/Layr-Labs/eigenda/api" - pb "github.com/Layr-Labs/eigenda/api/grpc/node" - "github.com/Layr-Labs/eigenda/core" - "github.com/Layr-Labs/eigenda/encoding" - "github.com/Layr-Labs/eigenda/node" - "github.com/consensys/gnark-crypto/ecc/bn254" - "github.com/consensys/gnark-crypto/ecc/bn254/fp" - "github.com/gammazero/workerpool" - "github.com/wealdtech/go-merkletree/v2" - "github.com/wealdtech/go-merkletree/v2/keccak256" - "google.golang.org/protobuf/proto" -) - -// GetBatchHeader constructs a core.BatchHeader from a proto of pb.StoreChunksRequest. -// Note the StoreChunksRequest is validated as soon as it enters the node gRPC -// interface, see grpc.Server.validateStoreChunkRequest. -func GetBatchHeader(in *pb.StoreChunksRequest) (*core.BatchHeader, error) { - var batchRoot [32]byte - copy(batchRoot[:], in.GetBatchHeader().GetBatchRoot()) - batchHeader := core.BatchHeader{ - ReferenceBlockNumber: uint(in.GetBatchHeader().GetReferenceBlockNumber()), - BatchRoot: batchRoot, - } - return &batchHeader, nil -} - -// GetBlobMessages constructs a core.BlobMessage array from blob protobufs. -// Note the proto request is validated as soon as it enters the node gRPC -// interface. This method assumes the blobs are valid. -func GetBlobMessages(pbBlobs []*pb.Blob, numWorkers int) ([]*core.BlobMessage, error) { - blobs := make([]*core.BlobMessage, len(pbBlobs)) - pool := workerpool.New(numWorkers) - resultChan := make(chan error, len(blobs)) - for i, blob := range pbBlobs { - i := i - blob := blob - pool.Submit(func() { - blobHeader, err := GetBlobHeaderFromProto(blob.GetHeader()) - - if err != nil { - resultChan <- err - return - } - if len(blob.GetBundles()) != len(blob.GetHeader().GetQuorumHeaders()) { - resultChan <- fmt.Errorf("number of quorum headers (%d) does not match number of bundles in blob message (%d)", len(blob.GetHeader().GetQuorumHeaders()), len(blob.GetBundles())) - return - } - - format := node.GetBundleEncodingFormat(blob) - bundles := make(map[core.QuorumID]core.Bundle, len(blob.GetBundles())) - for j, bundle := range blob.GetBundles() { - quorumID := blob.GetHeader().GetQuorumHeaders()[j].GetQuorumId() - if format == core.GnarkBundleEncodingFormat { - if len(bundle.GetBundle()) > 0 { - bundleMsg, err := new(core.Bundle).Deserialize(bundle.GetBundle()) - if err != nil { - resultChan <- err - return - } - bundles[uint8(quorumID)] = bundleMsg - } else { - bundles[uint8(quorumID)] = make([]*encoding.Frame, 0) - } - } else if format == core.GobBundleEncodingFormat { - bundles[uint8(quorumID)] = make([]*encoding.Frame, len(bundle.GetChunks())) - for k, data := range bundle.GetChunks() { - chunk, err := new(encoding.Frame).Deserialize(data) - if err != nil { - resultChan <- err - return - } - bundles[uint8(quorumID)][k] = chunk - } - } else { - resultChan <- fmt.Errorf("invalid bundle encoding format: %d", format) - return - } - } - - blobs[i] = &core.BlobMessage{ - BlobHeader: blobHeader, - Bundles: bundles, - } - - resultChan <- nil - }) - } - pool.StopWait() - close(resultChan) - for err := range resultChan { - if err != nil { - return nil, err - } - } - return blobs, nil -} - -func ValidatePointsFromBlobHeader(h *pb.BlobHeader) error { - commitX := new(fp.Element).SetBytes(h.GetCommitment().GetX()) - commitY := new(fp.Element).SetBytes(h.GetCommitment().GetY()) - commitment := &encoding.G1Commitment{ - X: *commitX, - Y: *commitY, - } - - if !(*bn254.G1Affine)(commitment).IsInSubGroup() { - return errors.New("commitment is not in the subgroup") - } - - var lengthCommitment, lengthProof encoding.G2Commitment - if h.GetLengthCommitment() != nil { - lengthCommitment.X.A0 = *new(fp.Element).SetBytes(h.GetLengthCommitment().GetXA0()) - lengthCommitment.X.A1 = *new(fp.Element).SetBytes(h.GetLengthCommitment().GetXA1()) - lengthCommitment.Y.A0 = *new(fp.Element).SetBytes(h.GetLengthCommitment().GetYA0()) - lengthCommitment.Y.A1 = *new(fp.Element).SetBytes(h.GetLengthCommitment().GetYA1()) - } - - if !(*bn254.G2Affine)(&lengthCommitment).IsInSubGroup() { - return errors.New("lengthCommitment is not in the subgroup") - } - - if h.GetLengthProof() != nil { - lengthProof.X.A0 = *new(fp.Element).SetBytes(h.GetLengthProof().GetXA0()) - lengthProof.X.A1 = *new(fp.Element).SetBytes(h.GetLengthProof().GetXA1()) - lengthProof.Y.A0 = *new(fp.Element).SetBytes(h.GetLengthProof().GetYA0()) - lengthProof.Y.A1 = *new(fp.Element).SetBytes(h.GetLengthProof().GetYA1()) - } - - if !(*bn254.G2Affine)(&lengthProof).IsInSubGroup() { - return errors.New("lengthProof is not in the subgroup") - } - return nil -} - -// GetBlobHeaderFromProto constructs a core.BlobHeader from a proto of pb.BlobHeader. -func GetBlobHeaderFromProto(h *pb.BlobHeader) (*core.BlobHeader, error) { - - if h == nil { - return nil, api.NewInvalidArgError("GetBlobHeaderFromProto: blob header is nil") - - } - - commitX := new(fp.Element).SetBytes(h.GetCommitment().GetX()) - commitY := new(fp.Element).SetBytes(h.GetCommitment().GetY()) - commitment := &encoding.G1Commitment{ - X: *commitX, - Y: *commitY, - } - - if !(*bn254.G1Affine)(commitment).IsInSubGroup() { - return nil, errors.New("commitment is not in the subgroup") - } - - var lengthCommitment, lengthProof encoding.G2Commitment - if h.GetLengthCommitment() != nil { - lengthCommitment.X.A0 = *new(fp.Element).SetBytes(h.GetLengthCommitment().GetXA0()) - lengthCommitment.X.A1 = *new(fp.Element).SetBytes(h.GetLengthCommitment().GetXA1()) - lengthCommitment.Y.A0 = *new(fp.Element).SetBytes(h.GetLengthCommitment().GetYA0()) - lengthCommitment.Y.A1 = *new(fp.Element).SetBytes(h.GetLengthCommitment().GetYA1()) - } - - if !(*bn254.G2Affine)(&lengthCommitment).IsInSubGroup() { - return nil, errors.New("lengthCommitment is not in the subgroup") - } - - if h.GetLengthProof() != nil { - lengthProof.X.A0 = *new(fp.Element).SetBytes(h.GetLengthProof().GetXA0()) - lengthProof.X.A1 = *new(fp.Element).SetBytes(h.GetLengthProof().GetXA1()) - lengthProof.Y.A0 = *new(fp.Element).SetBytes(h.GetLengthProof().GetYA0()) - lengthProof.Y.A1 = *new(fp.Element).SetBytes(h.GetLengthProof().GetYA1()) - } - - if !(*bn254.G2Affine)(&lengthProof).IsInSubGroup() { - return nil, errors.New("lengthProof is not in the subgroup") - } - - quorumHeaders := make([]*core.BlobQuorumInfo, len(h.GetQuorumHeaders())) - for i, header := range h.GetQuorumHeaders() { - if header.GetQuorumId() > core.MaxQuorumID { - return nil, api.NewInvalidArgError(fmt.Sprintf("quorum ID must be in range [0, %d], but found %d", core.MaxQuorumID, header.GetQuorumId())) - } - if err := core.ValidateSecurityParam(header.GetConfirmationThreshold(), header.GetAdversaryThreshold()); err != nil { - return nil, err - } - - quorumHeaders[i] = &core.BlobQuorumInfo{ - SecurityParam: core.SecurityParam{ - QuorumID: core.QuorumID(header.GetQuorumId()), - AdversaryThreshold: uint8(header.GetAdversaryThreshold()), - ConfirmationThreshold: uint8(header.GetConfirmationThreshold()), - QuorumRate: header.GetRatelimit(), - }, - ChunkLength: uint(header.GetChunkLength()), - } - } - - return &core.BlobHeader{ - BlobCommitments: encoding.BlobCommitments{ - Commitment: commitment, - LengthCommitment: &lengthCommitment, - LengthProof: &lengthProof, - Length: uint(h.GetLength()), - }, - QuorumInfos: quorumHeaders, - AccountID: h.AccountId, - }, nil -} - -// rebuildMerkleTree rebuilds the merkle tree from the blob headers and batch header. -func (s *Server) rebuildMerkleTree(batchHeaderHash [32]byte) (*merkletree.MerkleTree, error) { - batchHeaderBytes, err := s.node.Store.GetBatchHeader(context.Background(), batchHeaderHash) - if err != nil { - return nil, errors.New("failed to get the batch header from Store") - } - - batchHeader, err := new(core.BatchHeader).Deserialize(batchHeaderBytes) - if err != nil { - return nil, err - } - - blobIndex := 0 - leafs := make([][]byte, 0) - for { - blobHeaderBytes, err := s.node.Store.GetBlobHeader(context.Background(), batchHeaderHash, blobIndex) - if err != nil { - if errors.Is(err, node.ErrKeyNotFound) { - break - } - return nil, err - } - - var protoBlobHeader pb.BlobHeader - err = proto.Unmarshal(blobHeaderBytes, &protoBlobHeader) - if err != nil { - return nil, err - } - - blobHeader, err := GetBlobHeaderFromProto(&protoBlobHeader) - if err != nil { - return nil, err - } - - blobHeaderHash, err := blobHeader.GetBlobHeaderHash() - if err != nil { - return nil, err - } - leafs = append(leafs, blobHeaderHash[:]) - blobIndex++ - } - - if len(leafs) == 0 { - return nil, errors.New("no blob header found") - } - - tree, err := merkletree.NewTree(merkletree.WithData(leafs), merkletree.WithHashType(keccak256.New())) - if err != nil { - return nil, err - } - - if !reflect.DeepEqual(tree.Root(), batchHeader.BatchRoot[:]) { - return nil, errors.New("invalid batch header") - } - - return tree, nil -} - -// // Constructs a core.SecurityParam from a proto of pb.SecurityParams. -// func GetSecurityParam(p []*pb.SecurityParam) []*core.SecurityParam { -// res := make([]*core.SecurityParam, len(p)) -// for i := range p { -// res[i] = &core.SecurityParam{ -// QuorumID: core.QuorumID(p[i].GetQuorumId()), -// AdversaryThreshold: uint8(p[i].GetAdversaryThreshold()), -// } -// } -// return res -// } - -// // Constructs a core.QuorumParam array from a proto of pb.BatchHeader. -// func GetQuorumParams(p *pb.BatchHeader) []core.QuorumParam { -// quorum := make([]core.QuorumParam, 0) -// for _, param := range p.GetQuorumParams() { -// qp := core.QuorumParam{ -// QuorumID: core.QuorumID(param.GetQuorumId()), -// ConfirmationThreshold: uint8(param.GetQuorumThreshold()), -// } -// quorum = append(quorum, qp) -// } -// return quorum -// } diff --git a/node/node.go b/node/node.go index a90c593e81..865a78e74d 100644 --- a/node/node.go +++ b/node/node.go @@ -12,6 +12,7 @@ import ( "net/http" "net/url" "os" + "reflect" "strings" "sync" "time" @@ -21,6 +22,9 @@ import ( gethcommon "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" "github.com/prometheus/client_golang/prometheus" + "github.com/wealdtech/go-merkletree/v2" + "github.com/wealdtech/go-merkletree/v2/keccak256" + "google.golang.org/protobuf/proto" "github.com/Layr-Labs/eigenda/api/grpc/node" "github.com/Layr-Labs/eigenda/common/geth" @@ -262,8 +266,8 @@ func (n *Node) expireLoop() { // The heuristic is to cap the GC time to a percentage of the poll interval, but at // least have 1 second. timeLimitSec := uint64(math.Max(float64(n.Config.ExpirationPollIntervalSec)*gcPercentageTime, 1.0)) - numBatchesDeleted, numBlobsDeleted, err := n.Store.DeleteExpiredEntries(time.Now().Unix(), timeLimitSec) - n.Logger.Info("Complete an expiration cycle to remove expired batches", "num expired batches found and removed", numBatchesDeleted, "num expired blobs found and removed", numBlobsDeleted) + numBatchesDeleted, numMappingsDeleted, numBlobsDeleted, err := n.Store.DeleteExpiredEntries(time.Now().Unix(), timeLimitSec) + n.Logger.Info("Complete an expiration cycle to remove expired batches", "num expired batches found and removed", numBatchesDeleted, "num expired mappings found and removed", numMappingsDeleted, "num expired blobs found and removed", numBlobsDeleted) if err != nil { if errors.Is(err, context.DeadlineExceeded) { n.Logger.Error("Expiration cycle exited with ContextDeadlineExceed, meaning more expired batches need to be removed, which will continue in next cycle", "time limit (sec)", timeLimitSec) @@ -589,6 +593,57 @@ func (n *Node) SignBlobs(blobs []*core.BlobMessage, referenceBlockNumber uint) ( return signatures, nil } +// ValidateBlobHeadersRoot validates the blob headers root hash +// by comparing it with the merkle tree root hash of the blob headers. +// It also checks if all blob headers have the same reference block number +func (n *Node) ValidateBatchContents(ctx context.Context, blobHeaderHashes [][32]byte, batchHeader *core.BatchHeader) error { + leafs := make([][]byte, 0) + for _, blobHeaderHash := range blobHeaderHashes { + blobHeaderBytes, err := n.Store.GetBlobHeaderByHeaderHash(ctx, blobHeaderHash) + if err != nil { + return fmt.Errorf("failed to get blob header by hash: %w", err) + } + if blobHeaderBytes == nil { + return fmt.Errorf("blob header not found for hash %x", blobHeaderHash) + } + + var protoBlobHeader node.BlobHeader + err = proto.Unmarshal(blobHeaderBytes, &protoBlobHeader) + if err != nil { + return fmt.Errorf("failed to unmarshal blob header: %w", err) + } + if uint32(batchHeader.ReferenceBlockNumber) != protoBlobHeader.GetReferenceBlockNumber() { + return errors.New("blob headers have different reference block numbers") + } + + blobHeader, err := GetBlobHeaderFromProto(&protoBlobHeader) + if err != nil { + return fmt.Errorf("failed to get blob header from proto: %w", err) + } + + blobHeaderHash, err := blobHeader.GetBlobHeaderHash() + if err != nil { + return fmt.Errorf("failed to get blob header hash: %w", err) + } + leafs = append(leafs, blobHeaderHash[:]) + } + + if len(leafs) == 0 { + return errors.New("no blob headers found") + } + + tree, err := merkletree.NewTree(merkletree.WithData(leafs), merkletree.WithHashType(keccak256.New())) + if err != nil { + return fmt.Errorf("failed to create merkle tree: %w", err) + } + + if !reflect.DeepEqual(tree.Root(), batchHeader.BatchRoot[:]) { + return errors.New("invalid batch header") + } + + return nil +} + func (n *Node) updateSocketAddress(ctx context.Context, newSocketAddr string) { n.mu.Lock() defer n.mu.Unlock() diff --git a/node/store.go b/node/store.go index c0afb7111b..73fb7ae72f 100644 --- a/node/store.go +++ b/node/store.go @@ -66,32 +66,40 @@ func NewLevelDBStore(path string, logger logging.Logger, metrics *Metrics, block // The function returns the number of batches deleted and the status of deletion. Note that the // number of batches deleted can be positive even if the status is error (e.g. the error happened // after it had successfully deleted some batches). -func (s *Store) DeleteExpiredEntries(currentTimeUnixSec int64, timeLimitSec uint64) (numBatchesDeleted int, numBlobsDeleted int, err error) { +func (s *Store) DeleteExpiredEntries(currentTimeUnixSec int64, timeLimitSec uint64) (numBatchesDeleted int, numMappingsDeleted int, numBlobsDeleted int, err error) { ctx, cancel := context.WithTimeout(context.Background(), time.Duration(timeLimitSec)*time.Second) defer cancel() totalBatchesDeleted := 0 + totalMappingsDeleted := 0 totalBlobsDeleted := 0 for { select { case <-ctx.Done(): - return totalBatchesDeleted, totalBlobsDeleted, ctx.Err() + return totalBatchesDeleted, totalMappingsDeleted, totalBlobsDeleted, ctx.Err() default: blobsDeleted, err := s.deleteExpiredBlobs(currentTimeUnixSec, numBatchesToDeleteAtomically) if err != nil { - return totalBatchesDeleted, totalBlobsDeleted, err + return totalBatchesDeleted, totalMappingsDeleted, totalBlobsDeleted, err } totalBlobsDeleted += blobsDeleted batchesDeleted, err := s.deleteNBatches(currentTimeUnixSec, numBatchesToDeleteAtomically) if err != nil { - return totalBatchesDeleted, totalBlobsDeleted, err + return totalBatchesDeleted, totalMappingsDeleted, totalBlobsDeleted, err } totalBatchesDeleted += batchesDeleted + + mappingsDeleted, batchesDeleted, err := s.deleteExpiredBatchMapping(currentTimeUnixSec, numBatchesToDeleteAtomically) + if err != nil { + return totalBatchesDeleted, totalMappingsDeleted, totalBlobsDeleted, err + } + totalMappingsDeleted += mappingsDeleted + totalBatchesDeleted += batchesDeleted // When there is no error and we didn't delete any batch, it means we have // no obsolete batches to delete, so we can return. - if blobsDeleted == 0 && batchesDeleted == 0 { - return totalBatchesDeleted, totalBlobsDeleted, nil + if blobsDeleted == 0 && batchesDeleted == 0 && mappingsDeleted == 0 { + return totalBatchesDeleted, totalMappingsDeleted, totalBlobsDeleted, nil } } } @@ -109,7 +117,7 @@ func (s *Store) deleteExpiredBlobs(currentTimeUnixSec int64, numBlobs int) (int, for iter.Next() { ts, err := DecodeBlobExpirationKey(iter.Key()) if err != nil { - s.logger.Error("Could not decode the expiration key", "key:", iter.Key(), "error:", err) + s.logger.Error("Could not decode the expiration key", "key", iter.Key(), "error", err) continue } // No more rows expired up to current time. @@ -120,7 +128,7 @@ func (s *Store) deleteExpiredBlobs(currentTimeUnixSec int64, numBlobs int) (int, blobHeaderBytes := copyBytes(iter.Value()) blobHeaders, err := DecodeHashSlice(blobHeaderBytes) if err != nil { - s.logger.Error("Could not decode the blob header hashes", "error:", err) + s.logger.Error("Could not decode the blob header hashes", "error", err) continue } expiredBlobHeaders = append(expiredBlobHeaders, blobHeaders...) @@ -167,6 +175,67 @@ func (s *Store) deleteExpiredBlobs(currentTimeUnixSec int64, numBlobs int) (int, return len(expiredBlobHeaders), nil } +// deleteExpiredBatchMapping returns the number of batch to blob index mapping entries deleted and the status of deletion. +// The first return value is the number of batch to blob index mapping entries deleted. +// The second return value is the number of batch header entries deleted. +func (s *Store) deleteExpiredBatchMapping(currentTimeUnixSec int64, numBatches int) (numExpiredMappings int, numExpiredBatches int, err error) { + // Scan for expired batches. + iter := s.db.NewIterator(EncodeBatchMappingExpirationKeyPrefix()) + expiredKeys := make([][]byte, 0) + expiredBatches := make([][]byte, 0) + for iter.Next() { + ts, err := DecodeBatchMappingExpirationKey(iter.Key()) + if err != nil { + s.logger.Error("Could not decode the batch mapping expiration key", "key", iter.Key(), "error", err) + continue + } + // No more rows expired up to current time. + if currentTimeUnixSec < ts { + break + } + expiredKeys = append(expiredKeys, copyBytes(iter.Key())) + expiredBatches = append(expiredBatches, copyBytes(iter.Value())) + if len(expiredKeys) == numBatches { + break + } + } + iter.Release() + + // No expired batch found. + if len(expiredKeys) == 0 { + return 0, 0, nil + } + + numMappings := 0 + // Scan for the batch header, blob headers and chunks of each expired batch. + for _, hash := range expiredBatches { + var batchHeaderHash [32]byte + copy(batchHeaderHash[:], hash) + + // Batch header. + expiredKeys = append(expiredKeys, EncodeBatchHeaderKey(batchHeaderHash)) + + // Blob index mapping. + blobIndexIter := s.db.NewIterator(EncodeBlobIndexKeyPrefix(batchHeaderHash)) + for blobIndexIter.Next() { + expiredKeys = append(expiredKeys, copyBytes(blobIndexIter.Key())) + numMappings++ + } + blobIndexIter.Release() + } + + // Perform the removal. + err = s.db.DeleteBatch(expiredKeys) + if err != nil { + return -1, -1, fmt.Errorf("failed to delete the expired keys in batch: %w", err) + } + + s.logger.Info("Deleted expired batch mapping", "numBatches", len(expiredBatches), "numMappings", numMappings) + numExpiredMappings = numMappings + numExpiredBatches = len(expiredBatches) + return numExpiredMappings, numExpiredBatches, nil +} + // deleteNBatches returns the number of batches we deleted and the status of deletion. The number // is set to -1 (invalid value) if the deletion status is an error. func (s *Store) deleteNBatches(currentTimeUnixSec int64, numBatches int) (int, error) { @@ -177,7 +246,7 @@ func (s *Store) deleteNBatches(currentTimeUnixSec int64, numBatches int) (int, e for iter.Next() { ts, err := DecodeBatchExpirationKey(iter.Key()) if err != nil { - s.logger.Error("Could not decode the expiration key", "key:", iter.Key(), "error:", err) + s.logger.Error("Could not decode the expiration key", "key:", iter.Key(), "error", err) continue } // No more rows expired up to current time. @@ -228,7 +297,7 @@ func (s *Store) deleteNBatches(currentTimeUnixSec int64, numBatches int) (int, e // Perform the removal. err := s.db.DeleteBatch(expiredKeys) if err != nil { - s.logger.Error("Failed to delete the expired keys in batch", "keys:", expiredKeys, "error:", err) + s.logger.Error("Failed to delete the expired keys in batch", "keys", expiredKeys, "error", err) return -1, err } @@ -279,23 +348,7 @@ func (s *Store) StoreBatch(ctx context.Context, header *core.BatchHeader, blobs keys = append(keys, batchHeaderKey) values = append(values, batchHeaderBytes) - // Setting the expiration time for the batch. - curr := time.Now().Unix() - timeToExpire := (s.blockStaleMeasure + s.storeDurationBlocks) * 12 // 12s per block - // Why this expiration time is safe? - // - // The batch must be confirmed before referenceBlockNumber+blockStaleMeasure, otherwise - // it's stale and won't be accepted onchain. This means the blob's lifecycle will end - // before referenceBlockNumber+blockStaleMeasure+storeDurationBlocks. - // Since time@referenceBlockNumber < time.Now() (we always use a reference block that's - // already onchain), we have - // time@(referenceBlockNumber+blockStaleMeasure+storeDurationBlocks) - // = time@referenceBlockNumber + 12*(blockStaleMeasure+storeDurationBlocks) - // < time.Now() + 12*(blockStaleMeasure+storeDurationBlocks). - // - // Note if a batch is unconfirmed, it could be removed even earlier; here we treat its - // lifecycle the same as confirmed batches for simplicity. - expirationTime := curr + int64(timeToExpire) + expirationTime := s.expirationTime() expirationKey := EncodeBatchExpirationKey(expirationTime) keys = append(keys, expirationKey) values = append(values, batchHeaderHash[:]) @@ -404,23 +457,7 @@ func (s *Store) StoreBlobs(ctx context.Context, blobs []*core.BlobMessage, blobs keys := make([][]byte, 0) values := make([][]byte, 0) - // Setting the expiration time for the batch. - curr := time.Now().Unix() - timeToExpire := (s.blockStaleMeasure + s.storeDurationBlocks) * 12 // 12s per block - // Why this expiration time is safe? - // - // The batch must be confirmed before referenceBlockNumber+blockStaleMeasure, otherwise - // it's stale and won't be accepted onchain. This means the blob's lifecycle will end - // before referenceBlockNumber+blockStaleMeasure+storeDurationBlocks. - // Since time@referenceBlockNumber < time.Now() (we always use a reference block that's - // already onchain), we have - // time@(referenceBlockNumber+blockStaleMeasure+storeDurationBlocks) - // = time@referenceBlockNumber + 12*(blockStaleMeasure+storeDurationBlocks) - // < time.Now() + 12*(blockStaleMeasure+storeDurationBlocks). - // - // Note if a batch is unconfirmed, it could be removed even earlier; here we treat its - // lifecycle the same as confirmed batches for simplicity. - expirationTime := curr + int64(timeToExpire) + expirationTime := s.expirationTime() expirationKey := EncodeBlobExpirationKey(expirationTime) // expirationValue is a list of blob header hashes that are expired. expirationValue := make([]byte, 0) @@ -448,7 +485,6 @@ func (s *Store) StoreBlobs(ctx context.Context, blobs []*core.BlobMessage, blobs return nil, fmt.Errorf("failed to get blob header hash: %w", err) } blobHeaderKey := EncodeBlobHeaderKeyByHash(blobHeaderHash) - if s.HasKey(ctx, blobHeaderKey) { s.logger.Warn("Blob already exists", "blobHeaderHash", hexutil.Encode(blobHeaderHash[:])) continue @@ -539,6 +575,65 @@ func (s *Store) StoreBlobs(ctx context.Context, blobs []*core.BlobMessage, blobs return &keys, nil } +func (s *Store) StoreBatchBlobMapping(ctx context.Context, batchHeader *core.BatchHeader, blobHeaderHashes [][32]byte) error { + start := time.Now() + // The key/value pairs that need to be written to the local database. + keys := make([][]byte, 0) + values := make([][]byte, 0) + + batchHeaderHash, err := batchHeader.GetBatchHeaderHash() + if err != nil { + return fmt.Errorf("failed to get the batch header hash: %w", err) + } + + expirationTime := s.expirationTime() + expirationKey := EncodeBatchMappingExpirationKey(expirationTime, batchHeaderHash) + keys = append(keys, expirationKey) + values = append(values, batchHeaderHash[:]) + + for blobIndex, blobHeaderHash := range blobHeaderHashes { + blobIndexKey := EncodeBlobIndexKey(batchHeaderHash, blobIndex) + keys = append(keys, blobIndexKey) + values = append(values, copyBytes(blobHeaderHash[:])) + } + + // Generate the key/value pair for batch header. + batchHeaderKey := EncodeBatchHeaderKey(batchHeaderHash) + batchHeaderBytes, err := batchHeader.Serialize() + if err != nil { + return fmt.Errorf("failed to serialize the batch header: %w", err) + } + keys = append(keys, batchHeaderKey) + values = append(values, batchHeaderBytes) + + err = s.db.WriteBatch(keys, values) + if err != nil { + return fmt.Errorf("failed to write the blob index mappings into local database: %w", err) + } + s.logger.Debug("StoreBatchBlobMapping succeeded", "duration", time.Since(start)) + return nil +} + +func (s *Store) expirationTime() int64 { + // Setting the expiration time for the batch. + curr := time.Now().Unix() + timeToExpire := (s.blockStaleMeasure + s.storeDurationBlocks) * 12 // 12s per block + // Why this expiration time is safe? + // + // The batch must be confirmed before referenceBlockNumber+blockStaleMeasure, otherwise + // it's stale and won't be accepted onchain. This means the blob's lifecycle will end + // before referenceBlockNumber+blockStaleMeasure+storeDurationBlocks. + // Since time@referenceBlockNumber < time.Now() (we always use a reference block that's + // already onchain), we have + // time@(referenceBlockNumber+blockStaleMeasure+storeDurationBlocks) + // = time@referenceBlockNumber + 12*(blockStaleMeasure+storeDurationBlocks) + // < time.Now() + 12*(blockStaleMeasure+storeDurationBlocks). + // + // Note if a batch is unconfirmed, it could be removed even earlier; here we treat its + // lifecycle the same as confirmed batches for simplicity. + return curr + int64(timeToExpire) +} + // GetBatchHeader returns the batch header for the given batchHeaderHash. func (s *Store) GetBatchHeader(ctx context.Context, batchHeaderHash [32]byte) ([]byte, error) { batchHeaderKey := EncodeBatchHeaderKey(batchHeaderHash) @@ -604,6 +699,18 @@ func (s *Store) GetChunks(ctx context.Context, batchHeaderHash [32]byte, blobInd return chunks, format, nil } +func (s *Store) GetBlobHeaderHashAtIndex(ctx context.Context, batchHeaderHash [32]byte, blobIndex int) ([]byte, error) { + blobIndexKey := EncodeBlobIndexKey(batchHeaderHash, blobIndex) + data, err := s.db.Get(blobIndexKey) + if err != nil { + if errors.Is(err, leveldb.ErrNotFound) { + return nil, ErrKeyNotFound + } + return nil, err + } + return data, nil +} + // HasKey returns if a given key has been stored. func (s *Store) HasKey(ctx context.Context, key []byte) bool { _, err := s.db.Get(key) diff --git a/node/store_test.go b/node/store_test.go index 739990e04d..7c24787dfa 100644 --- a/node/store_test.go +++ b/node/store_test.go @@ -323,12 +323,12 @@ func TestStoreBatchSuccess(t *testing.T) { // Expire the batches. curTime := time.Now().Unix() + int64(staleMeasure+storeDuration)*12 // Try to expire at a time before expiry, so nothing will be expired. - numDeleted, _, err := s.DeleteExpiredEntries(curTime-10, 1) + numDeleted, _, _, err := s.DeleteExpiredEntries(curTime-10, 1) assert.Nil(t, err) assert.Equal(t, numDeleted, 0) assert.True(t, s.HasKey(ctx, batchHeaderKey)) // Then expire it at a time post expiry, so the batch will get purged. - numDeleted, _, err = s.DeleteExpiredEntries(curTime+10, 1) + numDeleted, _, _, err = s.DeleteExpiredEntries(curTime+10, 1) assert.Nil(t, err) assert.Equal(t, numDeleted, 1) assert.False(t, s.HasKey(ctx, batchHeaderKey)) @@ -384,15 +384,17 @@ func TestStoreBlobsSuccess(t *testing.T) { // Expire the batches. curTime := time.Now().Unix() + int64(staleMeasure+storeDuration)*12 // Try to expire at a time before expiry, so nothing will be expired. - numBatchesDeleted, numBlobsDeleted, err := s.DeleteExpiredEntries(curTime-10, 5) + numBatchesDeleted, numMappingsDeleted, numBlobsDeleted, err := s.DeleteExpiredEntries(curTime-10, 5) assert.Nil(t, err) assert.Equal(t, numBatchesDeleted, 0) + assert.Equal(t, numMappingsDeleted, 0) assert.Equal(t, numBlobsDeleted, 0) assert.True(t, s.HasKey(ctx, blobHeaderKey0)) // Then expire it at a time post expiry, so the batch will get purged. - numBatchesDeleted, numBlobsDeleted, err = s.DeleteExpiredEntries(curTime+10, 5) + numBatchesDeleted, numMappingsDeleted, numBlobsDeleted, err = s.DeleteExpiredEntries(curTime+10, 5) assert.Nil(t, err) assert.Equal(t, numBatchesDeleted, 0) + assert.Equal(t, numMappingsDeleted, 0) assert.Equal(t, numBlobsDeleted, 2) assert.False(t, s.HasKey(ctx, blobHeaderKey0)) assert.False(t, s.HasKey(ctx, blobHeaderKey1)) @@ -400,6 +402,70 @@ func TestStoreBlobsSuccess(t *testing.T) { assert.False(t, s.HasKey(ctx, blobKey1)) } +func TestStoreBatchBlobMapping(t *testing.T) { + s := createStore(t) + ctx := context.Background() + + // Empty store + blobKey := []byte{1, 2} + assert.False(t, s.HasKey(ctx, blobKey)) + + // Prepare data to store. + batchHeader, blobs, _ := CreateBatch(t) + batchHeaderHash, err := batchHeader.GetBatchHeaderHash() + assert.Nil(t, err) + blobHeaderHash0, err := blobs[0].BlobHeader.GetBlobHeaderHash() + assert.Nil(t, err) + blobHeaderHash1, err := blobs[1].BlobHeader.GetBlobHeaderHash() + assert.Nil(t, err) + + // Store a batch. + err = s.StoreBatchBlobMapping(ctx, batchHeader, [][32]byte{blobHeaderHash0, blobHeaderHash1}) + assert.Nil(t, err) + + // Check existence: batch header. + batchHeaderKey := node.EncodeBatchHeaderKey(batchHeaderHash) + assert.True(t, s.HasKey(ctx, batchHeaderKey)) + batchHeaderBytes, err := s.GetBatchHeader(ctx, batchHeaderHash) + assert.Nil(t, err) + expectedBatchHeaderBytes, err := batchHeader.Serialize() + assert.Nil(t, err) + assert.True(t, bytes.Equal(batchHeaderBytes, expectedBatchHeaderBytes)) + + // Check existence: blob index mapping + blobIndexKey0 := node.EncodeBlobIndexKey(batchHeaderHash, 0) + blobIndexKey1 := node.EncodeBlobIndexKey(batchHeaderHash, 1) + assert.True(t, s.HasKey(ctx, blobIndexKey0)) + assert.True(t, s.HasKey(ctx, blobIndexKey1)) + + var h [32]byte + bhh0, err := s.GetBlobHeaderHashAtIndex(ctx, batchHeaderHash, 0) + assert.Nil(t, err) + copy(h[:], bhh0) + assert.Equal(t, blobHeaderHash0, h) + bhh1, err := s.GetBlobHeaderHashAtIndex(ctx, batchHeaderHash, 1) + assert.Nil(t, err) + copy(h[:], bhh1) + assert.Equal(t, blobHeaderHash1, h) + + // Expire the batches. + curTime := time.Now().Unix() + int64(staleMeasure+storeDuration)*12 + // Try to expire at a time before expiry, so nothing will be expired. + numBatchesDeleted, numMappingsDeleted, _, err := s.DeleteExpiredEntries(curTime-10, 5) + assert.Nil(t, err) + assert.Equal(t, numBatchesDeleted, 0) + assert.Equal(t, numMappingsDeleted, 0) + assert.True(t, s.HasKey(ctx, blobIndexKey0)) + assert.True(t, s.HasKey(ctx, blobIndexKey1)) + // Then expire it at a time post expiry, so the batch will get purged. + numBatchesDeleted, numMappingsDeleted, _, err = s.DeleteExpiredEntries(curTime+10, 5) + assert.Nil(t, err) + assert.Equal(t, numBatchesDeleted, 1) + assert.Equal(t, numMappingsDeleted, 2) + assert.False(t, s.HasKey(ctx, blobIndexKey0)) + assert.False(t, s.HasKey(ctx, blobIndexKey1)) +} + func decodeChunks(t *testing.T, s *node.Store, batchHeaderHash [32]byte, blobIdx int, chunkEncoding pb.ChunkEncodingFormat) []*encoding.Frame { ctx := context.Background() chunks, format, err := s.GetChunks(ctx, batchHeaderHash, blobIdx, 0) diff --git a/node/store_utils.go b/node/store_utils.go new file mode 100644 index 0000000000..c6aa764e38 --- /dev/null +++ b/node/store_utils.go @@ -0,0 +1,197 @@ +package node + +import ( + "bytes" + "encoding/binary" + "errors" + + "github.com/Layr-Labs/eigenda/core" +) + +const ( + // Caution: the change to these prefixes needs to handle the backward compatibility, + // making sure the new code work with old data in DA Node store. + blobHeaderPrefix = "_BLOB_HEADER_" // The prefix of the blob header key. + batchHeaderPrefix = "_BATCH_HEADER_" // The prefix of the batch header key. + batchExpirationPrefix = "_EXPIRATION_" // The prefix of the batch expiration key. + // blobExpirationPrefix is the prefix of the blob and blob header expiration key. + // The blobs/blob headers expired by this prefix are those that are not associated with any batch. + // All blobs/blob headers in a batch are expired by the batch expiration key above. + blobExpirationPrefix = "_BLOBEXPIRATION_" + // batchMappingExpirationPrefix is the prefix of the batch mapping expiration key. + // This key is used to expire the batch to blob index mapping used to identify blob index in a full batch. + batchMappingExpirationPrefix = "_BATCHEXPIRATION_" + blobPrefix = "_BLOB_" // The prefix of the blob key. + blobIndexPrefix = "_BLOB_INDEX" // The prefix of the blob index key. +) + +// EncodeBlobKey returns an encoded key as blob identification. +func EncodeBlobKey(batchHeaderHash [32]byte, blobIndex int, quorumID core.QuorumID) ([]byte, error) { + buf := bytes.NewBuffer(batchHeaderHash[:]) + err := binary.Write(buf, binary.LittleEndian, int32(blobIndex)) + if err != nil { + return nil, err + } + err = binary.Write(buf, binary.LittleEndian, quorumID) + if err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +func EncodeBlobKeyByHash(blobHeaderHash [32]byte, quorumID core.QuorumID) ([]byte, error) { + prefix := []byte(blobHeaderPrefix) + buf := bytes.NewBuffer(append(prefix, blobHeaderHash[:]...)) + err := binary.Write(buf, binary.LittleEndian, quorumID) + if err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +func EncodeBlobKeyByHashPrefix(blobHeaderHash [32]byte) []byte { + prefix := []byte(blobHeaderPrefix) + buf := bytes.NewBuffer(append(prefix, blobHeaderHash[:]...)) + return buf.Bytes() +} + +// EncodeBlobHeaderKey returns an encoded key as blob header identification. +func EncodeBlobHeaderKey(batchHeaderHash [32]byte, blobIndex int) ([]byte, error) { + prefix := []byte(blobHeaderPrefix) + buf := bytes.NewBuffer(append(prefix, batchHeaderHash[:]...)) + err := binary.Write(buf, binary.LittleEndian, int32(blobIndex)) + if err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +// Returns an encoded prefix of blob header key. +func EncodeBlobHeaderKeyPrefix(batchHeaderHash [32]byte) []byte { + prefix := []byte(blobHeaderPrefix) + buf := bytes.NewBuffer(append(prefix, batchHeaderHash[:]...)) + return buf.Bytes() +} + +func EncodeBlobHeaderKeyByHash(blobHeaderHash [32]byte) []byte { + prefix := []byte(blobHeaderPrefix) + buf := bytes.NewBuffer(append(prefix, blobHeaderHash[:]...)) + return buf.Bytes() +} + +// EncodeBatchHeaderKey returns an encoded key as batch header identification. +func EncodeBatchHeaderKey(batchHeaderHash [32]byte) []byte { + prefix := []byte(batchHeaderPrefix) + buf := bytes.NewBuffer(append(prefix, batchHeaderHash[:]...)) + return buf.Bytes() +} + +func EncodeBlobIndexKey(batchHeaderHash [32]byte, blobIndex int) []byte { + prefix := []byte(blobIndexPrefix) + buf := bytes.NewBuffer(append(prefix, batchHeaderHash[:]...)) + err := binary.Write(buf, binary.LittleEndian, int32(blobIndex)) + if err != nil { + return nil + } + return buf.Bytes() +} + +func EncodeBlobIndexKeyPrefix(batchHeaderHash [32]byte) []byte { + prefix := []byte(blobIndexPrefix) + buf := bytes.NewBuffer(append(prefix, batchHeaderHash[:]...)) + return buf.Bytes() +} + +// EncodeBatchExpirationKeyPrefix returns the encoded prefix for batch expiration key. +func EncodeBatchExpirationKeyPrefix() []byte { + return []byte(batchExpirationPrefix) +} + +// EncodeBlobExpirationKeyPrefix returns the encoded prefix for blob expiration key. +func EncodeBlobExpirationKeyPrefix() []byte { + return []byte(blobExpirationPrefix) +} + +// EncodeBatchMappingExpirationKeyPrefix returns the encoded prefix for the expiration key of the batch to blob index mapping. +func EncodeBatchMappingExpirationKeyPrefix() []byte { + return []byte(batchMappingExpirationPrefix) +} + +// Returns an encoded key for expration time. +// Note: the encoded key will preserve the order of expiration time, that is, +// expirationTime1 < expirationTime2 <=> +// EncodeBatchExpirationKey(expirationTime1) < EncodeBatchExpirationKey(expirationTime2) +func EncodeBatchExpirationKey(expirationTime int64) []byte { + prefix := []byte(batchExpirationPrefix) + ts := make([]byte, 8) + binary.BigEndian.PutUint64(ts[0:8], uint64(expirationTime)) + buf := bytes.NewBuffer(append(prefix, ts[:]...)) + return buf.Bytes() +} + +// EncodeBlobExpirationKey returns an encoded key for expration time for blob header hashes. +// Note: the encoded key will preserve the order of expiration time, that is, +// expirationTime1 < expirationTime2 <=> +// EncodeBlobExpirationKey(expirationTime1) < EncodeBlobExpirationKey(expirationTime2) +func EncodeBlobExpirationKey(expirationTime int64) []byte { + prefix := []byte(blobExpirationPrefix) + ts := make([]byte, 8) + binary.BigEndian.PutUint64(ts[0:8], uint64(expirationTime)) + buf := bytes.NewBuffer(append(prefix, ts[:]...)) + return buf.Bytes() +} + +// EncodeBatchMappingExpirationKeyPrefix returns an encoded key for expration time for the batch to blob index mapping. +// Encodes the expiration time and the batch header hash into a single key. +// Note: the encoded key will preserve the order of expiration time, that is, +// expirationTime1 < expirationTime2 <=> +// EncodeBatchMappingExpirationKeyPrefix(expirationTime1) < EncodeBatchMappingExpirationKeyPrefix(expirationTime2) +func EncodeBatchMappingExpirationKey(expirationTime int64, batchHeaderHash [32]byte) []byte { + prefix := []byte(batchMappingExpirationPrefix) + ts := make([]byte, 8) + binary.BigEndian.PutUint64(ts[0:8], uint64(expirationTime)) + buf := bytes.NewBuffer(append(prefix, ts[:]...)) + buf.Write(batchHeaderHash[:]) + return buf.Bytes() +} + +// DecodeBatchExpirationKey returns the expiration timestamp encoded in the key. +func DecodeBatchExpirationKey(key []byte) (int64, error) { + if len(key) != len(batchExpirationPrefix)+8 { + return 0, errors.New("the expiration key is invalid") + } + ts := int64(binary.BigEndian.Uint64(key[len(key)-8:])) + return ts, nil +} + +// Returns the expiration timestamp encoded in the key. +func DecodeBlobExpirationKey(key []byte) (int64, error) { + if len(key) != len(blobExpirationPrefix)+8 { + return 0, errors.New("the expiration key is invalid") + } + ts := int64(binary.BigEndian.Uint64(key[len(key)-8:])) + return ts, nil +} + +// DecodeBatchMappingExpirationKey returns the expiration timestamp encoded in the key. +func DecodeBatchMappingExpirationKey(key []byte) (int64, error) { + if len(key) != len(batchMappingExpirationPrefix)+8+32 { + return 0, errors.New("the expiration key is invalid") + } + ts := int64(binary.BigEndian.Uint64(key[len(key)-8-32 : len(key)-32])) + return ts, nil +} + +func DecodeHashSlice(input []byte) ([][32]byte, error) { + if len(input)%32 != 0 { + return nil, errors.New("input length is not a multiple of 32") + } + numHashes := len(input) / 32 + + result := make([][32]byte, numHashes) + for i := 0; i < numHashes; i++ { + copy(result[i][:], input[i*32:(i+1)*32]) + } + + return result, nil +} diff --git a/node/store_utils_test.go b/node/store_utils_test.go new file mode 100644 index 0000000000..cc13cb339e --- /dev/null +++ b/node/store_utils_test.go @@ -0,0 +1,83 @@ +package node_test + +import ( + "os" + "testing" + + "github.com/Layr-Labs/eigenda/node" + "github.com/Layr-Labs/eigenda/node/leveldb" + "github.com/stretchr/testify/assert" +) + +func TestDecodeHashSlice(t *testing.T) { + hash0 := [32]byte{0, 1} + hash1 := [32]byte{0, 1, 2, 3, 4} + hash2 := [32]byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31} + + input := make([]byte, 0) + input = append(input, hash0[:]...) + input = append(input, hash1[:]...) + input = append(input, hash2[:]...) + + hashes, err := node.DecodeHashSlice(input) + assert.NoError(t, err) + assert.Len(t, hashes, 3) + assert.Equal(t, hash0, hashes[0]) + assert.Equal(t, hash1, hashes[1]) + assert.Equal(t, hash2, hashes[2]) +} + +func TestEncodeDecodeBatchMappingExpirationKey(t *testing.T) { + expirationTime := int64(1234567890) + batchHeaderHash := [32]byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31} + + key := node.EncodeBatchMappingExpirationKey(expirationTime, batchHeaderHash) + decodedExpirationTime, err := node.DecodeBatchMappingExpirationKey(key) + assert.NoError(t, err) + assert.Equal(t, expirationTime, decodedExpirationTime) +} + +func TestBatchMappingExpirationKeyOrdering(t *testing.T) { + dbPath := t.TempDir() + defer os.Remove(dbPath) + + db, err := leveldb.NewLevelDBStore(dbPath) + assert.NoError(t, err) + + keys := make([][]byte, 0) + values := make([][]byte, 0) + // test ordering using expiration time + expirationTime := int64(1111111111) + batchHeaderHash := [32]byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31} + key := node.EncodeBatchMappingExpirationKey(expirationTime, batchHeaderHash) + keys = append(keys, key) + values = append(values, []byte("value")) + + expirationTime = int64(2222222222) + batchHeaderHash = [32]byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31} + key = node.EncodeBatchMappingExpirationKey(expirationTime, batchHeaderHash) + keys = append(keys, key) + values = append(values, []byte("value")) + + expirationTime = int64(3333333333) + batchHeaderHash = [32]byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31} + key = node.EncodeBatchMappingExpirationKey(expirationTime, batchHeaderHash) + keys = append(keys, key) + values = append(values, []byte("value")) + + err = db.WriteBatch(keys, values) + assert.NoError(t, err) + + iter := db.NewIterator(node.EncodeBatchMappingExpirationKeyPrefix()) + assert.NoError(t, err) + defer iter.Release() + i := 0 + expectedExpirationTimes := []int64{1111111111, 2222222222, 3333333333} + for iter.Next() { + ts, err := node.DecodeBatchMappingExpirationKey(iter.Key()) + assert.NoError(t, err) + assert.Equal(t, expectedExpirationTimes[i], ts) + i++ + } + assert.Equal(t, 3, i) +} diff --git a/node/utils.go b/node/utils.go index 5e37d1cf9b..91722e6055 100644 --- a/node/utils.go +++ b/node/utils.go @@ -1,155 +1,216 @@ package node import ( - "bytes" "context" - "encoding/binary" "errors" "fmt" + "github.com/Layr-Labs/eigenda/api" pb "github.com/Layr-Labs/eigenda/api/grpc/node" "github.com/Layr-Labs/eigenda/common/pubip" "github.com/Layr-Labs/eigenda/core" + "github.com/Layr-Labs/eigenda/encoding" + "github.com/consensys/gnark-crypto/ecc/bn254" + "github.com/consensys/gnark-crypto/ecc/bn254/fp" + "github.com/gammazero/workerpool" ) -const ( - // Caution: the change to these prefixes needs to handle the backward compatibility, - // making sure the new code work with old data in DA Node store. - blobHeaderPrefix = "_BLOB_HEADER_" // The prefix of the blob header key. - batchHeaderPrefix = "_BATCH_HEADER_" // The prefix of the batch header key. - batchExpirationPrefix = "_EXPIRATION_" // The prefix of the batch expiration key. - // blobExpirationPrefix is the prefix of the blob and blob header expiration key. - // The blobs/blob headers expired by this prefix are those that are not associated with any batch. - // All blobs/blob headers in a batch are expired by the batch expiration key above. - blobExpirationPrefix = "_BLOBEXPIRATION_" - blobPrefix = "_BLOB_" // The prefix of the blob key. -) - -// EncodeBlobKey returns an encoded key as blob identification. -func EncodeBlobKey(batchHeaderHash [32]byte, blobIndex int, quorumID core.QuorumID) ([]byte, error) { - buf := bytes.NewBuffer(batchHeaderHash[:]) - err := binary.Write(buf, binary.LittleEndian, int32(blobIndex)) - if err != nil { - return nil, err +// GetBatchHeader constructs a core.BatchHeader from a proto of pb.StoreChunksRequest. +// Note the StoreChunksRequest is validated as soon as it enters the node gRPC +// interface, see grpc.Server.validateStoreChunkRequest. +func GetBatchHeader(in *pb.BatchHeader) (*core.BatchHeader, error) { + if in == nil || len(in.GetBatchRoot()) == 0 { + return nil, api.NewInvalidArgError("batch header is nil or empty") } - err = binary.Write(buf, binary.LittleEndian, quorumID) - if err != nil { - return nil, err + var batchRoot [32]byte + copy(batchRoot[:], in.GetBatchRoot()) + batchHeader := core.BatchHeader{ + ReferenceBlockNumber: uint(in.GetReferenceBlockNumber()), + BatchRoot: batchRoot, } - return buf.Bytes(), nil + return &batchHeader, nil } -func EncodeBlobKeyByHash(blobHeaderHash [32]byte, quorumID core.QuorumID) ([]byte, error) { - prefix := []byte(blobHeaderPrefix) - buf := bytes.NewBuffer(append(prefix, blobHeaderHash[:]...)) - err := binary.Write(buf, binary.LittleEndian, quorumID) - if err != nil { - return nil, err +// GetBlobMessages constructs a core.BlobMessage array from blob protobufs. +// Note the proto request is validated as soon as it enters the node gRPC +// interface. This method assumes the blobs are valid. +func GetBlobMessages(pbBlobs []*pb.Blob, numWorkers int) ([]*core.BlobMessage, error) { + blobs := make([]*core.BlobMessage, len(pbBlobs)) + pool := workerpool.New(numWorkers) + resultChan := make(chan error, len(blobs)) + for i, blob := range pbBlobs { + i := i + blob := blob + pool.Submit(func() { + blobHeader, err := GetBlobHeaderFromProto(blob.GetHeader()) + + if err != nil { + resultChan <- err + return + } + if len(blob.GetBundles()) != len(blob.GetHeader().GetQuorumHeaders()) { + resultChan <- fmt.Errorf("number of quorum headers (%d) does not match number of bundles in blob message (%d)", len(blob.GetHeader().GetQuorumHeaders()), len(blob.GetBundles())) + return + } + + format := GetBundleEncodingFormat(blob) + bundles := make(map[core.QuorumID]core.Bundle, len(blob.GetBundles())) + for j, bundle := range blob.GetBundles() { + quorumID := blob.GetHeader().GetQuorumHeaders()[j].GetQuorumId() + if format == core.GnarkBundleEncodingFormat { + if len(bundle.GetBundle()) > 0 { + bundleMsg, err := new(core.Bundle).Deserialize(bundle.GetBundle()) + if err != nil { + resultChan <- err + return + } + bundles[uint8(quorumID)] = bundleMsg + } else { + bundles[uint8(quorumID)] = make([]*encoding.Frame, 0) + } + } else if format == core.GobBundleEncodingFormat { + bundles[uint8(quorumID)] = make([]*encoding.Frame, len(bundle.GetChunks())) + for k, data := range bundle.GetChunks() { + chunk, err := new(encoding.Frame).Deserialize(data) + if err != nil { + resultChan <- err + return + } + bundles[uint8(quorumID)][k] = chunk + } + } else { + resultChan <- fmt.Errorf("invalid bundle encoding format: %d", format) + return + } + } + + blobs[i] = &core.BlobMessage{ + BlobHeader: blobHeader, + Bundles: bundles, + } + + resultChan <- nil + }) + } + pool.StopWait() + close(resultChan) + for err := range resultChan { + if err != nil { + return nil, err + } } - return buf.Bytes(), nil + return blobs, nil } -func EncodeBlobKeyByHashPrefix(blobHeaderHash [32]byte) []byte { - prefix := []byte(blobHeaderPrefix) - buf := bytes.NewBuffer(append(prefix, blobHeaderHash[:]...)) - return buf.Bytes() -} +func ValidatePointsFromBlobHeader(h *pb.BlobHeader) error { + commitX := new(fp.Element).SetBytes(h.GetCommitment().GetX()) + commitY := new(fp.Element).SetBytes(h.GetCommitment().GetY()) + commitment := &encoding.G1Commitment{ + X: *commitX, + Y: *commitY, + } -// EncodeBlobHeaderKey returns an encoded key as blob header identification. -func EncodeBlobHeaderKey(batchHeaderHash [32]byte, blobIndex int) ([]byte, error) { - prefix := []byte(blobHeaderPrefix) - buf := bytes.NewBuffer(append(prefix, batchHeaderHash[:]...)) - err := binary.Write(buf, binary.LittleEndian, int32(blobIndex)) - if err != nil { - return nil, err + if !(*bn254.G1Affine)(commitment).IsInSubGroup() { + return errors.New("commitment is not in the subgroup") } - return buf.Bytes(), nil -} -// Returns an encoded prefix of blob header key. -func EncodeBlobHeaderKeyPrefix(batchHeaderHash [32]byte) []byte { - prefix := []byte(blobHeaderPrefix) - buf := bytes.NewBuffer(append(prefix, batchHeaderHash[:]...)) - return buf.Bytes() -} + var lengthCommitment, lengthProof encoding.G2Commitment + if h.GetLengthCommitment() != nil { + lengthCommitment.X.A0 = *new(fp.Element).SetBytes(h.GetLengthCommitment().GetXA0()) + lengthCommitment.X.A1 = *new(fp.Element).SetBytes(h.GetLengthCommitment().GetXA1()) + lengthCommitment.Y.A0 = *new(fp.Element).SetBytes(h.GetLengthCommitment().GetYA0()) + lengthCommitment.Y.A1 = *new(fp.Element).SetBytes(h.GetLengthCommitment().GetYA1()) + } -func EncodeBlobHeaderKeyByHash(blobHeaderHash [32]byte) []byte { - prefix := []byte(blobHeaderPrefix) - buf := bytes.NewBuffer(append(prefix, blobHeaderHash[:]...)) - return buf.Bytes() -} + if !(*bn254.G2Affine)(&lengthCommitment).IsInSubGroup() { + return errors.New("lengthCommitment is not in the subgroup") + } -// EncodeBatchHeaderKey returns an encoded key as batch header identification. -func EncodeBatchHeaderKey(batchHeaderHash [32]byte) []byte { - prefix := []byte(batchHeaderPrefix) - buf := bytes.NewBuffer(append(prefix, batchHeaderHash[:]...)) - return buf.Bytes() -} + if h.GetLengthProof() != nil { + lengthProof.X.A0 = *new(fp.Element).SetBytes(h.GetLengthProof().GetXA0()) + lengthProof.X.A1 = *new(fp.Element).SetBytes(h.GetLengthProof().GetXA1()) + lengthProof.Y.A0 = *new(fp.Element).SetBytes(h.GetLengthProof().GetYA0()) + lengthProof.Y.A1 = *new(fp.Element).SetBytes(h.GetLengthProof().GetYA1()) + } -// Returns the encoded prefix for batch expiration key. -func EncodeBatchExpirationKeyPrefix() []byte { - return []byte(batchExpirationPrefix) + if !(*bn254.G2Affine)(&lengthProof).IsInSubGroup() { + return errors.New("lengthProof is not in the subgroup") + } + return nil } -// Returns the encoded prefix for blob expiration key. -func EncodeBlobExpirationKeyPrefix() []byte { - return []byte(blobExpirationPrefix) -} +// GetBlobHeaderFromProto constructs a core.BlobHeader from a proto of pb.BlobHeader. +func GetBlobHeaderFromProto(h *pb.BlobHeader) (*core.BlobHeader, error) { -// Returns an encoded key for expration time. -// Note: the encoded key will preserve the order of expiration time, that is, -// expirationTime1 < expirationTime2 <=> -// EncodeBatchExpirationKey(expirationTime1) < EncodeBatchExpirationKey(expirationTime2) -func EncodeBatchExpirationKey(expirationTime int64) []byte { - prefix := []byte(batchExpirationPrefix) - ts := make([]byte, 8) - binary.BigEndian.PutUint64(ts[0:8], uint64(expirationTime)) - buf := bytes.NewBuffer(append(prefix, ts[:]...)) - return buf.Bytes() -} + if h == nil { + return nil, api.NewInvalidArgError("GetBlobHeaderFromProto: blob header is nil") -// Returns an encoded key for expration time for blob header hashes. -// Note: the encoded key will preserve the order of expiration time, that is, -// expirationTime1 < expirationTime2 <=> -// EncodeBlobExpirationKey(expirationTime1) < EncodeBlobExpirationKey(expirationTime2) -func EncodeBlobExpirationKey(expirationTime int64) []byte { - prefix := []byte(blobExpirationPrefix) - ts := make([]byte, 8) - binary.BigEndian.PutUint64(ts[0:8], uint64(expirationTime)) - buf := bytes.NewBuffer(append(prefix, ts[:]...)) - return buf.Bytes() -} + } -// Returns the expiration timestamp encoded in the key. -func DecodeBatchExpirationKey(key []byte) (int64, error) { - if len(key) != len(batchExpirationPrefix)+8 { - return 0, errors.New("the expiration key is invalid") + commitX := new(fp.Element).SetBytes(h.GetCommitment().GetX()) + commitY := new(fp.Element).SetBytes(h.GetCommitment().GetY()) + commitment := &encoding.G1Commitment{ + X: *commitX, + Y: *commitY, } - ts := int64(binary.BigEndian.Uint64(key[len(key)-8:])) - return ts, nil -} -// Returns the expiration timestamp encoded in the key. -func DecodeBlobExpirationKey(key []byte) (int64, error) { - if len(key) != len(blobExpirationPrefix)+8 { - return 0, errors.New("the expiration key is invalid") + if !(*bn254.G1Affine)(commitment).IsInSubGroup() { + return nil, errors.New("commitment is not in the subgroup") + } + + var lengthCommitment, lengthProof encoding.G2Commitment + if h.GetLengthCommitment() != nil { + lengthCommitment.X.A0 = *new(fp.Element).SetBytes(h.GetLengthCommitment().GetXA0()) + lengthCommitment.X.A1 = *new(fp.Element).SetBytes(h.GetLengthCommitment().GetXA1()) + lengthCommitment.Y.A0 = *new(fp.Element).SetBytes(h.GetLengthCommitment().GetYA0()) + lengthCommitment.Y.A1 = *new(fp.Element).SetBytes(h.GetLengthCommitment().GetYA1()) } - ts := int64(binary.BigEndian.Uint64(key[len(key)-8:])) - return ts, nil -} -func DecodeHashSlice(input []byte) ([][32]byte, error) { - if len(input)%32 != 0 { - return nil, errors.New("input length is not a multiple of 32") + if !(*bn254.G2Affine)(&lengthCommitment).IsInSubGroup() { + return nil, errors.New("lengthCommitment is not in the subgroup") } - numHashes := len(input) / 32 - result := make([][32]byte, numHashes) - for i := 0; i < numHashes; i++ { - copy(result[i][:], input[i*32:(i+1)*32]) + if h.GetLengthProof() != nil { + lengthProof.X.A0 = *new(fp.Element).SetBytes(h.GetLengthProof().GetXA0()) + lengthProof.X.A1 = *new(fp.Element).SetBytes(h.GetLengthProof().GetXA1()) + lengthProof.Y.A0 = *new(fp.Element).SetBytes(h.GetLengthProof().GetYA0()) + lengthProof.Y.A1 = *new(fp.Element).SetBytes(h.GetLengthProof().GetYA1()) } - return result, nil + if !(*bn254.G2Affine)(&lengthProof).IsInSubGroup() { + return nil, errors.New("lengthProof is not in the subgroup") + } + + quorumHeaders := make([]*core.BlobQuorumInfo, len(h.GetQuorumHeaders())) + for i, header := range h.GetQuorumHeaders() { + if header.GetQuorumId() > core.MaxQuorumID { + return nil, api.NewInvalidArgError(fmt.Sprintf("quorum ID must be in range [0, %d], but found %d", core.MaxQuorumID, header.GetQuorumId())) + } + if err := core.ValidateSecurityParam(header.GetConfirmationThreshold(), header.GetAdversaryThreshold()); err != nil { + return nil, err + } + + quorumHeaders[i] = &core.BlobQuorumInfo{ + SecurityParam: core.SecurityParam{ + QuorumID: core.QuorumID(header.GetQuorumId()), + AdversaryThreshold: uint8(header.GetAdversaryThreshold()), + ConfirmationThreshold: uint8(header.GetConfirmationThreshold()), + QuorumRate: header.GetRatelimit(), + }, + ChunkLength: uint(header.GetChunkLength()), + } + } + + return &core.BlobHeader{ + BlobCommitments: encoding.BlobCommitments{ + Commitment: commitment, + LengthCommitment: &lengthCommitment, + LengthProof: &lengthProof, + Length: uint(h.GetLength()), + }, + QuorumInfos: quorumHeaders, + AccountID: h.AccountId, + }, nil } func SocketAddress(ctx context.Context, provider pubip.Provider, dispersalPort string, retrievalPort string) (string, error) { @@ -175,3 +236,28 @@ func GetBundleEncodingFormat(blob *pb.Blob) core.BundleEncodingFormat { } return core.GobBundleEncodingFormat } + +// // Constructs a core.SecurityParam from a proto of pb.SecurityParams. +// func GetSecurityParam(p []*pb.SecurityParam) []*core.SecurityParam { +// res := make([]*core.SecurityParam, len(p)) +// for i := range p { +// res[i] = &core.SecurityParam{ +// QuorumID: core.QuorumID(p[i].GetQuorumId()), +// AdversaryThreshold: uint8(p[i].GetAdversaryThreshold()), +// } +// } +// return res +// } + +// // Constructs a core.QuorumParam array from a proto of pb.BatchHeader. +// func GetQuorumParams(p *pb.BatchHeader) []core.QuorumParam { +// quorum := make([]core.QuorumParam, 0) +// for _, param := range p.GetQuorumParams() { +// qp := core.QuorumParam{ +// QuorumID: core.QuorumID(param.GetQuorumId()), +// ConfirmationThreshold: uint8(param.GetQuorumThreshold()), +// } +// quorum = append(quorum, qp) +// } +// return quorum +// } diff --git a/node/utils_test.go b/node/utils_test.go deleted file mode 100644 index d555d449d2..0000000000 --- a/node/utils_test.go +++ /dev/null @@ -1,26 +0,0 @@ -package node_test - -import ( - "testing" - - "github.com/Layr-Labs/eigenda/node" - "github.com/stretchr/testify/assert" -) - -func TestDecodeHashSlice(t *testing.T) { - hash0 := [32]byte{0, 1} - hash1 := [32]byte{0, 1, 2, 3, 4} - hash2 := [32]byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31} - - input := make([]byte, 0) - input = append(input, hash0[:]...) - input = append(input, hash1[:]...) - input = append(input, hash2[:]...) - - hashes, err := node.DecodeHashSlice(input) - assert.NoError(t, err) - assert.Len(t, hashes, 3) - assert.Equal(t, hash0, hashes[0]) - assert.Equal(t, hash1, hashes[1]) - assert.Equal(t, hash2, hashes[2]) -} diff --git a/test/integration_test.go b/test/integration_test.go index 2d462e2981..6ce9f8bf69 100644 --- a/test/integration_test.go +++ b/test/integration_test.go @@ -544,7 +544,7 @@ func TestDispersalAndRetrieval(t *testing.T) { assert.Greater(t, headerReply.GetBlobHeader().GetQuorumHeaders()[0].GetChunkLength(), uint32(0)) if blobHeader == nil { - blobHeader, err = nodegrpc.GetBlobHeaderFromProto(headerReply.GetBlobHeader()) + blobHeader, err = node.GetBlobHeaderFromProto(headerReply.GetBlobHeader()) assert.NoError(t, err) } From f07e364d039c8a79af02ebbeab79e9d63ab54b22 Mon Sep 17 00:00:00 2001 From: Cody Littley <56973212+cody-littley@users.noreply.github.com> Date: Fri, 16 Aug 2024 12:06:00 -0500 Subject: [PATCH 4/5] Split blob writer code out of larger PR. (#685) Signed-off-by: Cody Littley --- tools/traffic/workers/blob_writer.go | 162 ++++++++++++++++++++++ tools/traffic/workers/blob_writer_test.go | 134 ++++++++++++++++++ tools/traffic/workers/key_handler.go | 7 + tools/traffic/workers/mock_disperser.go | 44 ++++++ tools/traffic/workers/mock_key_handler.go | 24 ++++ 5 files changed, 371 insertions(+) create mode 100644 tools/traffic/workers/blob_writer.go create mode 100644 tools/traffic/workers/blob_writer_test.go create mode 100644 tools/traffic/workers/key_handler.go create mode 100644 tools/traffic/workers/mock_disperser.go create mode 100644 tools/traffic/workers/mock_key_handler.go diff --git a/tools/traffic/workers/blob_writer.go b/tools/traffic/workers/blob_writer.go new file mode 100644 index 0000000000..a30a7e5bdd --- /dev/null +++ b/tools/traffic/workers/blob_writer.go @@ -0,0 +1,162 @@ +package workers + +import ( + "context" + "crypto/md5" + "crypto/rand" + "fmt" + "github.com/Layr-Labs/eigenda/api/clients" + "github.com/Layr-Labs/eigenda/encoding/utils/codec" + "github.com/Layr-Labs/eigenda/tools/traffic/config" + "github.com/Layr-Labs/eigenda/tools/traffic/metrics" + "github.com/Layr-Labs/eigensdk-go/logging" + "sync" + "time" +) + +// BlobWriter sends blobs to a disperser at a configured rate. +type BlobWriter struct { + // The context for the generator. All work should cease when this context is cancelled. + ctx *context.Context + + // Tracks the number of active goroutines within the generator. + waitGroup *sync.WaitGroup + + // All logs should be written using this logger. + logger logging.Logger + + // Config contains the configuration for the generator. + config *config.WorkerConfig + + // disperser is the client used to send blobs to the disperser. + disperser clients.DisperserClient + + // Unconfirmed keys are sent here. + unconfirmedKeyHandler KeyHandler + + // fixedRandomData contains random data for blobs if RandomizeBlobs is false, and nil otherwise. + fixedRandomData []byte + + // writeLatencyMetric is used to record latency for write requests. + writeLatencyMetric metrics.LatencyMetric + + // writeSuccessMetric is used to record the number of successful write requests. + writeSuccessMetric metrics.CountMetric + + // writeFailureMetric is used to record the number of failed write requests. + writeFailureMetric metrics.CountMetric +} + +// NewBlobWriter creates a new BlobWriter instance. +func NewBlobWriter( + ctx *context.Context, + waitGroup *sync.WaitGroup, + logger logging.Logger, + config *config.WorkerConfig, + disperser clients.DisperserClient, + unconfirmedKeyHandler KeyHandler, + generatorMetrics metrics.Metrics) BlobWriter { + + var fixedRandomData []byte + if config.RandomizeBlobs { + // New random data will be generated for each blob. + fixedRandomData = nil + } else { + // Use this random data for each blob. + fixedRandomData = make([]byte, config.DataSize) + _, err := rand.Read(fixedRandomData) + if err != nil { + panic(fmt.Sprintf("unable to read random data: %s", err)) + } + fixedRandomData = codec.ConvertByPaddingEmptyByte(fixedRandomData) + } + + return BlobWriter{ + ctx: ctx, + waitGroup: waitGroup, + logger: logger, + config: config, + disperser: disperser, + unconfirmedKeyHandler: unconfirmedKeyHandler, + fixedRandomData: fixedRandomData, + writeLatencyMetric: generatorMetrics.NewLatencyMetric("write"), + writeSuccessMetric: generatorMetrics.NewCountMetric("write_success"), + writeFailureMetric: generatorMetrics.NewCountMetric("write_failure"), + } +} + +// Start begins the blob writer goroutine. +func (writer *BlobWriter) Start() { + writer.waitGroup.Add(1) + ticker := time.NewTicker(writer.config.WriteRequestInterval) + + go func() { + defer writer.waitGroup.Done() + + for { + select { + case <-(*writer.ctx).Done(): + return + case <-ticker.C: + writer.writeNextBlob() + } + } + }() +} + +// writeNextBlob attempts to send a random blob to the disperser. +func (writer *BlobWriter) writeNextBlob() { + data, err := writer.getRandomData() + if err != nil { + writer.logger.Error("failed to get random data", "err", err) + return + } + key, err := metrics.InvokeAndReportLatency(writer.writeLatencyMetric, func() ([]byte, error) { + return writer.sendRequest(data) + }) + if err != nil { + writer.writeFailureMetric.Increment() + writer.logger.Error("failed to send blob request", "err", err) + return + } + + writer.writeSuccessMetric.Increment() + + checksum := md5.Sum(data) + writer.unconfirmedKeyHandler.AddUnconfirmedKey(key, checksum, uint(len(data))) +} + +// getRandomData returns a slice of random data to be used for a blob. +func (writer *BlobWriter) getRandomData() ([]byte, error) { + if writer.fixedRandomData != nil { + return writer.fixedRandomData, nil + } + + data := make([]byte, writer.config.DataSize) + _, err := rand.Read(data) + if err != nil { + return nil, fmt.Errorf("unable to read random data: %w", err) + } + data = codec.ConvertByPaddingEmptyByte(data) + + return data, nil +} + +// sendRequest sends a blob to a disperser. +func (writer *BlobWriter) sendRequest(data []byte) (key []byte, err error) { + ctxTimeout, cancel := context.WithTimeout(*writer.ctx, writer.config.WriteTimeout) + defer cancel() + + if writer.config.SignerPrivateKey != "" { + _, key, err = writer.disperser.DisperseBlobAuthenticated( + ctxTimeout, + data, + writer.config.CustomQuorums) + } else { + _, key, err = writer.disperser.DisperseBlob( + ctxTimeout, + data, + writer.config.CustomQuorums) + } + return +} diff --git a/tools/traffic/workers/blob_writer_test.go b/tools/traffic/workers/blob_writer_test.go new file mode 100644 index 0000000000..723894490a --- /dev/null +++ b/tools/traffic/workers/blob_writer_test.go @@ -0,0 +1,134 @@ +package workers + +import ( + "context" + "crypto/md5" + "fmt" + "github.com/Layr-Labs/eigenda/common" + tu "github.com/Layr-Labs/eigenda/common/testutils" + "github.com/Layr-Labs/eigenda/disperser" + "github.com/Layr-Labs/eigenda/encoding/utils/codec" + "github.com/Layr-Labs/eigenda/tools/traffic/config" + "github.com/Layr-Labs/eigenda/tools/traffic/metrics" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "golang.org/x/exp/rand" + "sync" + "testing" +) + +func TestBlobWriter(t *testing.T) { + tu.InitializeRandom() + + ctx, cancel := context.WithCancel(context.Background()) + waitGroup := sync.WaitGroup{} + logger, err := common.NewLogger(common.DefaultLoggerConfig()) + assert.Nil(t, err) + + dataSize := rand.Uint64()%1024 + 64 + + authenticated := rand.Intn(2) == 0 + var signerPrivateKey string + if authenticated { + signerPrivateKey = "asdf" + } + var functionName string + if authenticated { + functionName = "DisperseBlobAuthenticated" + } else { + functionName = "DisperseBlob" + } + + randomizeBlobs := rand.Intn(2) == 0 + + useCustomQuorum := rand.Intn(2) == 0 + var customQuorum []uint8 + if useCustomQuorum { + customQuorum = []uint8{1, 2, 3} + } + + config := &config.WorkerConfig{ + DataSize: dataSize, + SignerPrivateKey: signerPrivateKey, + RandomizeBlobs: randomizeBlobs, + CustomQuorums: customQuorum, + } + + disperserClient := &MockDisperserClient{} + unconfirmedKeyHandler := &MockKeyHandler{} + unconfirmedKeyHandler.mock.On( + "AddUnconfirmedKey", mock.Anything, mock.Anything, mock.Anything).Return(nil) + + generatorMetrics := metrics.NewMockMetrics() + + writer := NewBlobWriter( + &ctx, + &waitGroup, + logger, + config, + disperserClient, + unconfirmedKeyHandler, + generatorMetrics) + + errorCount := 0 + + var previousData []byte + + for i := 0; i < 100; i++ { + var errorToReturn error + if i%10 == 0 { + errorToReturn = fmt.Errorf("intentional error for testing purposes") + errorCount++ + } else { + errorToReturn = nil + } + + // This is the key that will be assigned to the next blob. + keyToReturn := make([]byte, 32) + _, err = rand.Read(keyToReturn) + assert.Nil(t, err) + + status := disperser.Processing + disperserClient.mock = mock.Mock{} // reset mock state + disperserClient.mock.On(functionName, mock.Anything, customQuorum).Return(&status, keyToReturn, errorToReturn) + + // Simulate the advancement of time (i.e. allow the writer to write the next blob). + writer.writeNextBlob() + + disperserClient.mock.AssertNumberOfCalls(t, functionName, 1) + unconfirmedKeyHandler.mock.AssertNumberOfCalls(t, "AddUnconfirmedKey", i+1-errorCount) + + if errorToReturn == nil { + + dataSentToDisperser := disperserClient.mock.Calls[0].Arguments.Get(0).([]byte) + assert.NotNil(t, dataSentToDisperser) + + // Strip away the extra encoding bytes. We should have data of the expected size. + decodedData := codec.RemoveEmptyByteFromPaddedBytes(dataSentToDisperser) + assert.Equal(t, dataSize, uint64(len(decodedData))) + + // Verify that the proper data was sent to the unconfirmed key handler. + checksum := md5.Sum(dataSentToDisperser) + + unconfirmedKeyHandler.mock.AssertCalled(t, "AddUnconfirmedKey", keyToReturn, checksum, uint(len(dataSentToDisperser))) + + // Verify that data has the proper amount of randomness. + if previousData != nil { + if randomizeBlobs { + // We expect each blob to be different. + assert.NotEqual(t, previousData, dataSentToDisperser) + } else { + // We expect each blob to be the same. + assert.Equal(t, previousData, dataSentToDisperser) + } + } + previousData = dataSentToDisperser + } + + // Verify metrics. + assert.Equal(t, float64(i+1-errorCount), generatorMetrics.GetCount("write_success")) + assert.Equal(t, float64(errorCount), generatorMetrics.GetCount("write_failure")) + } + + cancel() +} diff --git a/tools/traffic/workers/key_handler.go b/tools/traffic/workers/key_handler.go new file mode 100644 index 0000000000..30c8b5ed9c --- /dev/null +++ b/tools/traffic/workers/key_handler.go @@ -0,0 +1,7 @@ +package workers + +// KeyHandler is an interface describing an object that can accept unconfirmed keys. +type KeyHandler interface { + // AddUnconfirmedKey accepts an unconfirmed blob key, the checksum of the blob, and the size of the blob in bytes. + AddUnconfirmedKey(key []byte, checksum [16]byte, size uint) +} diff --git a/tools/traffic/workers/mock_disperser.go b/tools/traffic/workers/mock_disperser.go new file mode 100644 index 0000000000..ba1b013880 --- /dev/null +++ b/tools/traffic/workers/mock_disperser.go @@ -0,0 +1,44 @@ +package workers + +import ( + "context" + "github.com/Layr-Labs/eigenda/api/clients" + disperser_rpc "github.com/Layr-Labs/eigenda/api/grpc/disperser" + "github.com/Layr-Labs/eigenda/disperser" + "github.com/stretchr/testify/mock" +) + +var _ clients.DisperserClient = (*MockDisperserClient)(nil) + +type MockDisperserClient struct { + mock mock.Mock +} + +func (m *MockDisperserClient) DisperseBlob( + ctx context.Context, + data []byte, + customQuorums []uint8) (*disperser.BlobStatus, []byte, error) { + + args := m.mock.Called(data, customQuorums) + + return args.Get(0).(*disperser.BlobStatus), args.Get(1).([]byte), args.Error(2) +} + +func (m *MockDisperserClient) DisperseBlobAuthenticated( + ctx context.Context, + data []byte, + customQuorums []uint8) (*disperser.BlobStatus, []byte, error) { + + args := m.mock.Called(data, customQuorums) + return args.Get(0).(*disperser.BlobStatus), args.Get(1).([]byte), args.Error(2) +} + +func (m *MockDisperserClient) GetBlobStatus(ctx context.Context, key []byte) (*disperser_rpc.BlobStatusReply, error) { + args := m.mock.Called(key) + return args.Get(0).(*disperser_rpc.BlobStatusReply), args.Error(1) +} + +func (m *MockDisperserClient) RetrieveBlob(ctx context.Context, batchHeaderHash []byte, blobIndex uint32) ([]byte, error) { + args := m.mock.Called(batchHeaderHash, blobIndex) + return args.Get(0).([]byte), args.Error(1) +} diff --git a/tools/traffic/workers/mock_key_handler.go b/tools/traffic/workers/mock_key_handler.go new file mode 100644 index 0000000000..2c48de995b --- /dev/null +++ b/tools/traffic/workers/mock_key_handler.go @@ -0,0 +1,24 @@ +package workers + +import ( + "github.com/stretchr/testify/mock" +) + +var _ KeyHandler = (*MockKeyHandler)(nil) + +// MockKeyHandler is a stand-in for the blob verifier's UnconfirmedKeyHandler. +type MockKeyHandler struct { + mock mock.Mock + + ProvidedKey []byte + ProvidedChecksum [16]byte + ProvidedSize uint +} + +func (m *MockKeyHandler) AddUnconfirmedKey(key []byte, checksum [16]byte, size uint) { + m.mock.Called(key, checksum, size) + + m.ProvidedKey = key + m.ProvidedChecksum = checksum + m.ProvidedSize = size +} From cacdc21d65e8634ab3ced4e6c2dcb6ef0fdd2515 Mon Sep 17 00:00:00 2001 From: Jian Xiao <99709935+jianoaix@users.noreply.github.com> Date: Mon, 19 Aug 2024 11:14:05 -0700 Subject: [PATCH 5/5] [2/N][zero serialization] Make Batcher operate on chunks without ser/deser (#700) --- api/clients/mock/node_client.go | 11 +- api/clients/retrieval_client_test.go | 10 +- core/data.go | 90 ++++++++++++++- core/data_test.go | 122 ++++++++++++++------ core/test/core_test.go | 31 +++-- disperser/batcher/batcher_test.go | 2 +- disperser/batcher/encoded_blob_store.go | 7 +- disperser/batcher/encoding_streamer.go | 32 ++--- disperser/batcher/encoding_streamer_test.go | 48 ++++---- disperser/batcher/grpc/dispatcher.go | 55 ++++++--- disperser/batcher/minibatcher.go | 8 +- disperser/disperser.go | 2 +- disperser/encoder/client.go | 18 +-- disperser/encoder_client.go | 3 +- disperser/local_encoder_client.go | 19 ++- disperser/mock/dispatcher.go | 2 +- disperser/mock/encoder.go | 7 +- node/grpc/server_load_test.go | 23 ++-- 18 files changed, 351 insertions(+), 139 deletions(-) diff --git a/api/clients/mock/node_client.go b/api/clients/mock/node_client.go index 42228ae95d..6e13ee61b3 100644 --- a/api/clients/mock/node_client.go +++ b/api/clients/mock/node_client.go @@ -54,9 +54,18 @@ func (c *MockNodeClient) GetChunks( ) { args := c.Called(opID, opInfo, batchHeaderHash, blobIndex) encodedBlob := (args.Get(0)).(core.EncodedBlob) + chunks, err := encodedBlob.EncodedBundlesByOperator[opID][quorumID].ToFrames() + if err != nil { + chunksChan <- clients.RetrievedChunks{ + OperatorID: opID, + Err: err, + Chunks: nil, + } + + } chunksChan <- clients.RetrievedChunks{ OperatorID: opID, Err: nil, - Chunks: encodedBlob.BundlesByOperator[opID][quorumID], + Chunks: chunks, } } diff --git a/api/clients/retrieval_client_test.go b/api/clients/retrieval_client_test.go index bd2e12f752..2eb1b72f40 100644 --- a/api/clients/retrieval_client_test.go +++ b/api/clients/retrieval_client_test.go @@ -60,8 +60,8 @@ var ( retrievalClient clients.RetrievalClient blobHeader *core.BlobHeader encodedBlob core.EncodedBlob = core.EncodedBlob{ - BlobHeader: nil, - BundlesByOperator: make(map[core.OperatorID]core.Bundles), + BlobHeader: nil, + EncodedBundlesByOperator: make(map[core.OperatorID]core.EncodedBundles), } batchHeaderHash [32]byte batchRoot [32]byte @@ -198,7 +198,11 @@ func setup(t *testing.T) { bundles := make(map[core.QuorumID]core.Bundle, len(blobHeader.QuorumInfos)) bundles[quorumID] = chunks[assignment.StartIndex : assignment.StartIndex+assignment.NumChunks] encodedBlob.BlobHeader = blobHeader - encodedBlob.BundlesByOperator[id] = bundles + eb, err := core.Bundles(bundles).ToEncodedBundles() + if err != nil { + t.Fatal(err) + } + encodedBlob.EncodedBundlesByOperator[id] = eb } } diff --git a/core/data.go b/core/data.go index 7829ef3ad6..45f9e750f8 100644 --- a/core/data.go +++ b/core/data.go @@ -87,6 +87,49 @@ func (cd *ChunksData) Size() uint64 { return size } +func (cd *ChunksData) FromFrames(fr []*encoding.Frame) (*ChunksData, error) { + if len(fr) == 0 { + return nil, errors.New("no frame is provided") + } + var c ChunksData + c.Format = GnarkChunkEncodingFormat + c.ChunkLen = fr[0].Length() + c.Chunks = make([][]byte, 0, len(fr)) + for _, f := range fr { + bytes, err := f.SerializeGnark() + if err != nil { + return nil, err + } + c.Chunks = append(c.Chunks, bytes) + } + return &c, nil +} + +func (cd *ChunksData) ToFrames() ([]*encoding.Frame, error) { + frames := make([]*encoding.Frame, 0, len(cd.Chunks)) + switch cd.Format { + case GobChunkEncodingFormat: + for _, data := range cd.Chunks { + fr, err := new(encoding.Frame).Deserialize(data) + if err != nil { + return nil, err + } + frames = append(frames, fr) + } + case GnarkChunkEncodingFormat: + for _, data := range cd.Chunks { + fr, err := new(encoding.Frame).DeserializeGnark(data) + if err != nil { + return nil, err + } + frames = append(frames, fr) + } + default: + return nil, fmt.Errorf("invalid chunk encoding format: %v", cd.Format) + } + return frames, nil +} + func (cd *ChunksData) FlattenToBundle() ([]byte, error) { // Only Gnark coded chunks are dispersed as a byte array. // Gob coded chunks are not flattened. @@ -128,8 +171,9 @@ func (cd *ChunksData) ToGobFormat() (*ChunksData, error) { gobChunks = append(gobChunks, gob) } return &ChunksData{ - Chunks: gobChunks, - Format: GobChunkEncodingFormat, + Chunks: gobChunks, + Format: GobChunkEncodingFormat, + ChunkLen: cd.ChunkLen, }, nil } @@ -153,8 +197,9 @@ func (cd *ChunksData) ToGnarkFormat() (*ChunksData, error) { gnarkChunks = append(gnarkChunks, gnark) } return &ChunksData{ - Chunks: gnarkChunks, - Format: GnarkChunkEncodingFormat, + Chunks: gnarkChunks, + Format: GnarkChunkEncodingFormat, + ChunkLen: cd.ChunkLen, }, nil } @@ -266,6 +311,8 @@ type BatchHeader struct { type EncodedBlob struct { BlobHeader *BlobHeader BundlesByOperator map[OperatorID]Bundles + // EncodedBundlesByOperator is bundles in encoded format (not deserialized) + EncodedBundlesByOperator map[OperatorID]EncodedBundles } // A Bundle is the collection of chunks associated with a single blob, for a single operator and a single quorum. @@ -274,12 +321,23 @@ type Bundle []*encoding.Frame // Bundles is the collection of bundles associated with a single blob and a single operator. type Bundles map[QuorumID]Bundle +// This is similar to Bundle, but tracks chunks in encoded format (i.e. not deserialized). +type EncodedBundles map[QuorumID]*ChunksData + // BlobMessage is the message that is sent to DA nodes. It contains the blob header and the associated chunk bundles. type BlobMessage struct { BlobHeader *BlobHeader Bundles Bundles } +// This is similar to BlobMessage, but keep the commitments and chunks in encoded format +// (i.e. not deserialized) +type EncodedBlobMessage struct { + // TODO(jianoaix): Change the commitments to encoded format. + BlobHeader *BlobHeader + EncodedBundles map[QuorumID]*ChunksData +} + func (b Bundle) Size() uint64 { size := uint64(0) for _, chunk := range b { @@ -388,3 +446,27 @@ func (cb Bundles) Size() uint64 { } return size } + +func (cb Bundles) ToEncodedBundles() (EncodedBundles, error) { + eb := make(EncodedBundles) + for quorum, bundle := range cb { + cd, err := new(ChunksData).FromFrames(bundle) + if err != nil { + return nil, err + } + eb[quorum] = cd + } + return eb, nil +} + +func (cb Bundles) FromEncodedBundles(eb EncodedBundles) (Bundles, error) { + c := make(Bundles) + for quorum, chunkData := range eb { + fr, err := chunkData.ToFrames() + if err != nil { + return nil, err + } + c[quorum] = fr + } + return c, nil +} diff --git a/core/data_test.go b/core/data_test.go index c062c72442..84cb5097e9 100644 --- a/core/data_test.go +++ b/core/data_test.go @@ -33,6 +33,52 @@ func createBundle(t *testing.T, numFrames, numCoeffs, seed int) core.Bundle { return frames } +func createChunksData(t *testing.T, seed int) (core.Bundle, *core.ChunksData, *core.ChunksData) { + bundle := createBundle(t, 64, 64, seed) + gobChunks := make([][]byte, len(bundle)) + gnarkChunks := make([][]byte, len(bundle)) + for i, frame := range bundle { + gobChunk, err := frame.Serialize() + assert.Nil(t, err) + gobChunks[i] = gobChunk + + gnarkChunk, err := frame.SerializeGnark() + assert.Nil(t, err) + gnarkChunks[i] = gnarkChunk + } + gob := &core.ChunksData{ + Chunks: gobChunks, + Format: core.GobChunkEncodingFormat, + ChunkLen: 64, + } + gnark := &core.ChunksData{ + Chunks: gnarkChunks, + Format: core.GnarkChunkEncodingFormat, + ChunkLen: 64, + } + return bundle, gob, gnark +} + +func checkChunksDataEquivalence(t *testing.T, cd1, cd2 *core.ChunksData) { + assert.Equal(t, cd1.Format, cd2.Format) + assert.Equal(t, cd1.ChunkLen, cd2.ChunkLen) + assert.Equal(t, len(cd1.Chunks), len(cd2.Chunks)) + for i, c1 := range cd1.Chunks { + assert.True(t, bytes.Equal(c1, cd2.Chunks[i])) + } +} + +func checkBundleEquivalence(t *testing.T, b1, b2 core.Bundle) { + assert.Equal(t, len(b1), len(b2)) + for i := 0; i < len(b1); i++ { + assert.True(t, b1[i].Proof.Equal(&b2[i].Proof)) + assert.Equal(t, len(b1[i].Coeffs), len(b2[i].Coeffs)) + for j := 0; j < len(b1[i].Coeffs); j++ { + assert.True(t, b1[i].Coeffs[j].Equal(&b2[i].Coeffs[j])) + } + } +} + func TestInvalidBundleSer(t *testing.T) { b1 := createBundle(t, 1, 0, 0) _, err := b1.Serialize() @@ -86,41 +132,38 @@ func TestBundleEncoding(t *testing.T) { assert.Nil(t, err) decoded, err := new(core.Bundle).Deserialize(bytes) assert.Nil(t, err) - assert.Equal(t, len(bundle), len(decoded)) - for i := 0; i < len(bundle); i++ { - assert.True(t, bundle[i].Proof.Equal(&decoded[i].Proof)) - assert.Equal(t, len(bundle[i].Coeffs), len(decoded[i].Coeffs)) - for j := 0; j < len(bundle[i].Coeffs); j++ { - assert.True(t, bundle[i].Coeffs[j].Equal(&decoded[i].Coeffs[j])) - } - } + checkBundleEquivalence(t, bundle, decoded) } } -func createChunksData(t *testing.T, seed int) (core.Bundle, *core.ChunksData, *core.ChunksData) { - bundle := createBundle(t, 64, 64, seed) - gobChunks := make([][]byte, len(bundle)) - gnarkChunks := make([][]byte, len(bundle)) - for i, frame := range bundle { - gobChunk, err := frame.Serialize() +func TestEncodedBundles(t *testing.T) { + numTrials := 16 + for i := 0; i < numTrials; i++ { + bundles := core.Bundles(map[core.QuorumID]core.Bundle{ + 0: createBundle(t, 64, 64, i), + 1: createBundle(t, 64, 64, i+numTrials), + }) + // ToEncodedBundles + ec, err := bundles.ToEncodedBundles() assert.Nil(t, err) - gobChunks[i] = gobChunk - - gnarkChunk, err := frame.SerializeGnark() + assert.Equal(t, len(ec), len(bundles)) + for quorum, bundle := range bundles { + cd, ok := ec[quorum] + assert.True(t, ok) + fr, err := cd.ToFrames() + assert.Nil(t, err) + checkBundleEquivalence(t, fr, bundle) + } + // FromEncodedBundles + bundles2, err := new(core.Bundles).FromEncodedBundles(ec) assert.Nil(t, err) - gnarkChunks[i] = gnarkChunk - } - gob := &core.ChunksData{ - Chunks: gobChunks, - Format: core.GobChunkEncodingFormat, - ChunkLen: 64, - } - gnark := &core.ChunksData{ - Chunks: gnarkChunks, - Format: core.GnarkChunkEncodingFormat, - ChunkLen: 64, + assert.Equal(t, len(bundles2), len(bundles)) + for quorum, bundle := range bundles { + b, ok := bundles2[quorum] + assert.True(t, ok) + checkBundleEquivalence(t, b, bundle) + } } - return bundle, gob, gnark } func TestChunksData(t *testing.T) { @@ -136,26 +179,31 @@ func TestChunksData(t *testing.T) { assert.Equal(t, convertedGob, gob) convertedGob, err = gnark.ToGobFormat() assert.Nil(t, err) - assert.Equal(t, len(gob.Chunks), len(convertedGob.Chunks)) - for i := 0; i < len(gob.Chunks); i++ { - assert.True(t, bytes.Equal(gob.Chunks[i], convertedGob.Chunks[i])) - } + checkChunksDataEquivalence(t, gob, convertedGob) // ToGnarkFormat convertedGnark, err := gnark.ToGnarkFormat() assert.Nil(t, err) assert.Equal(t, convertedGnark, gnark) convertedGnark, err = gob.ToGnarkFormat() assert.Nil(t, err) - assert.Equal(t, len(gnark.Chunks), len(convertedGnark.Chunks)) - for i := 0; i < len(gnark.Chunks); i++ { - assert.True(t, bytes.Equal(gnark.Chunks[i], convertedGnark.Chunks[i])) - } + checkChunksDataEquivalence(t, gnark, convertedGnark) // FlattenToBundle bytesFromChunksData, err := gnark.FlattenToBundle() assert.Nil(t, err) bytesFromBundle, err := bundle.Serialize() assert.Nil(t, err) assert.True(t, bytes.Equal(bytesFromChunksData, bytesFromBundle)) + // FromFrames + cd, err := new(core.ChunksData).FromFrames(bundle) + assert.Nil(t, err) + checkChunksDataEquivalence(t, cd, gnark) + // ToFrames + fr1, err := gob.ToFrames() + assert.Nil(t, err) + checkBundleEquivalence(t, bundle, fr1) + fr2, err := gnark.ToFrames() + assert.Nil(t, err) + checkBundleEquivalence(t, bundle, fr2) // Invalid cases gnark.Chunks[0] = gnark.Chunks[0][1:] _, err = gnark.FlattenToBundle() diff --git a/core/test/core_test.go b/core/test/core_test.go index 04dd74a79e..9ee1ec6b10 100644 --- a/core/test/core_test.go +++ b/core/test/core_test.go @@ -113,8 +113,8 @@ func prepareBatch(t *testing.T, operatorCount uint, blobs []core.Blob, bn uint) blobHeaders[z] = blobHeader encodedBlob := core.EncodedBlob{ - BlobHeader: blobHeader, - BundlesByOperator: make(map[core.OperatorID]core.Bundles), + BlobHeader: blobHeader, + EncodedBundlesByOperator: make(map[core.OperatorID]core.EncodedBundles), } encodedBlobs[z] = encodedBlob @@ -156,6 +156,14 @@ func prepareBatch(t *testing.T, operatorCount uint, blobs []core.Blob, bn uint) if err != nil { t.Fatal(err) } + bytes := make([][]byte, 0, len(chunks)) + for _, c := range chunks { + serialized, err := c.Serialize() + if err != nil { + t.Fatal(err) + } + bytes = append(bytes, serialized) + } blobHeader.BlobCommitments = encoding.BlobCommitments{ Commitment: commitments.Commitment, @@ -167,13 +175,18 @@ func prepareBatch(t *testing.T, operatorCount uint, blobs []core.Blob, bn uint) blobHeader.QuorumInfos = append(blobHeader.QuorumInfos, quorumHeader) for id, assignment := range assignments { - _, ok := encodedBlob.BundlesByOperator[id] + chunksData := &core.ChunksData{ + Format: core.GobChunkEncodingFormat, + ChunkLen: int(chunkLength), + Chunks: bytes[assignment.StartIndex : assignment.StartIndex+assignment.NumChunks], + } + _, ok := encodedBlob.EncodedBundlesByOperator[id] if !ok { - encodedBlob.BundlesByOperator[id] = map[core.QuorumID]core.Bundle{ - quorumID: chunks[assignment.StartIndex : assignment.StartIndex+assignment.NumChunks], + encodedBlob.EncodedBundlesByOperator[id] = map[core.QuorumID]*core.ChunksData{ + quorumID: chunksData, } } else { - encodedBlob.BundlesByOperator[id][quorumID] = chunks[assignment.StartIndex : assignment.StartIndex+assignment.NumChunks] + encodedBlob.EncodedBundlesByOperator[id][quorumID] = chunksData } } @@ -207,9 +220,13 @@ func checkBatchByUniversalVerifier(cst core.IndexedChainState, encodedBlobs []co val.UpdateOperatorID(id) blobMessages := make([]*core.BlobMessage, numBlob) for z, encodedBlob := range encodedBlobs { + bundles, err := new(core.Bundles).FromEncodedBundles(encodedBlob.EncodedBundlesByOperator[id]) + if err != nil { + return err + } blobMessages[z] = &core.BlobMessage{ BlobHeader: encodedBlob.BlobHeader, - Bundles: encodedBlob.BundlesByOperator[id], + Bundles: bundles, } } err := val.ValidateBatch(&header, blobMessages, state.OperatorState, pool) diff --git a/disperser/batcher/batcher_test.go b/disperser/batcher/batcher_test.go index 29786e9205..0a2e8d91a7 100644 --- a/disperser/batcher/batcher_test.go +++ b/disperser/batcher/batcher_test.go @@ -227,7 +227,7 @@ func TestBatcherIterations(t *testing.T) { assert.NoError(t, err) count, size := components.encodingStreamer.EncodedBlobstore.GetEncodedResultSize() assert.Equal(t, 2, count) - assert.Equal(t, uint64(24576), size) // Robert checks it + assert.Equal(t, uint64(27631), size) txn := types.NewTransaction(0, gethcommon.Address{}, big.NewInt(0), 0, big.NewInt(0), nil) components.transactor.On("BuildConfirmBatchTxn", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Run(func(args mock.Arguments) { diff --git a/disperser/batcher/encoded_blob_store.go b/disperser/batcher/encoded_blob_store.go index 5ce98d6d71..6deccee54c 100644 --- a/disperser/batcher/encoded_blob_store.go +++ b/disperser/batcher/encoded_blob_store.go @@ -29,7 +29,7 @@ type EncodingResult struct { ReferenceBlockNumber uint BlobQuorumInfo *core.BlobQuorumInfo Commitment *encoding.BlobCommitments - Chunks []*encoding.Frame + ChunksData *core.ChunksData Assignments map[core.OperatorID]core.Assignment } @@ -197,5 +197,8 @@ func getRequestID(key disperser.BlobKey, quorumID core.QuorumID) requestID { // getChunksSize returns the total size of all the chunks in the encoded result in bytes func getChunksSize(result *EncodingResult) uint64 { - return core.Bundle(result.Chunks).Size() + if result == nil || result.ChunksData == nil { + return 0 + } + return result.ChunksData.Size() } diff --git a/disperser/batcher/encoding_streamer.go b/disperser/batcher/encoding_streamer.go index 3b22a3f829..99173f6f83 100644 --- a/disperser/batcher/encoding_streamer.go +++ b/disperser/batcher/encoding_streamer.go @@ -386,7 +386,7 @@ func (e *EncodingStreamer) RequestEncodingForBlob(ctx context.Context, metadata ReferenceBlockNumber: referenceBlockNumber, BlobQuorumInfo: res.BlobQuorumInfo, Commitment: commits, - Chunks: chunks, + ChunksData: chunks, Assignments: res.Assignments, }, Err: nil, @@ -482,19 +482,22 @@ func (e *EncodingStreamer) CreateMinibatch(ctx context.Context) (*batch, error) } blobHeaderByKey[blobKey] = blobHeader encodedBlobByKey[blobKey] = core.EncodedBlob{ - BlobHeader: blobHeader, - BundlesByOperator: make(map[core.OperatorID]core.Bundles), + BlobHeader: blobHeader, + EncodedBundlesByOperator: make(map[core.OperatorID]core.EncodedBundles), } } // Populate the assigned bundles for opID, assignment := range result.Assignments { - bundles, ok := encodedBlobByKey[blobKey].BundlesByOperator[opID] + bundles, ok := encodedBlobByKey[blobKey].EncodedBundlesByOperator[opID] if !ok { - encodedBlobByKey[blobKey].BundlesByOperator[opID] = make(core.Bundles) - bundles = encodedBlobByKey[blobKey].BundlesByOperator[opID] + encodedBlobByKey[blobKey].EncodedBundlesByOperator[opID] = make(core.EncodedBundles) + bundles = encodedBlobByKey[blobKey].EncodedBundlesByOperator[opID] } - bundles[result.BlobQuorumInfo.QuorumID] = append(bundles[result.BlobQuorumInfo.QuorumID], result.Chunks[assignment.StartIndex:assignment.StartIndex+assignment.NumChunks]...) + bundles[result.BlobQuorumInfo.QuorumID] = new(core.ChunksData) + bundles[result.BlobQuorumInfo.QuorumID].Format = result.ChunksData.Format + bundles[result.BlobQuorumInfo.QuorumID].Chunks = append(bundles[result.BlobQuorumInfo.QuorumID].Chunks, result.ChunksData.Chunks[assignment.StartIndex:assignment.StartIndex+assignment.NumChunks]...) + bundles[result.BlobQuorumInfo.QuorumID].ChunkLen = result.ChunksData.ChunkLen } blobQuorums[blobKey] = append(blobQuorums[blobKey], result.BlobQuorumInfo) @@ -631,19 +634,22 @@ func (e *EncodingStreamer) CreateBatch(ctx context.Context) (*batch, error) { } blobHeaderByKey[blobKey] = blobHeader encodedBlobByKey[blobKey] = core.EncodedBlob{ - BlobHeader: blobHeader, - BundlesByOperator: make(map[core.OperatorID]core.Bundles), + BlobHeader: blobHeader, + EncodedBundlesByOperator: make(map[core.OperatorID]core.EncodedBundles), } } // Populate the assigned bundles for opID, assignment := range result.Assignments { - bundles, ok := encodedBlobByKey[blobKey].BundlesByOperator[opID] + bundles, ok := encodedBlobByKey[blobKey].EncodedBundlesByOperator[opID] if !ok { - encodedBlobByKey[blobKey].BundlesByOperator[opID] = make(core.Bundles) - bundles = encodedBlobByKey[blobKey].BundlesByOperator[opID] + encodedBlobByKey[blobKey].EncodedBundlesByOperator[opID] = make(core.EncodedBundles) + bundles = encodedBlobByKey[blobKey].EncodedBundlesByOperator[opID] } - bundles[result.BlobQuorumInfo.QuorumID] = append(bundles[result.BlobQuorumInfo.QuorumID], result.Chunks[assignment.StartIndex:assignment.StartIndex+assignment.NumChunks]...) + bundles[result.BlobQuorumInfo.QuorumID] = new(core.ChunksData) + bundles[result.BlobQuorumInfo.QuorumID].Format = result.ChunksData.Format + bundles[result.BlobQuorumInfo.QuorumID].Chunks = append(bundles[result.BlobQuorumInfo.QuorumID].Chunks, result.ChunksData.Chunks[assignment.StartIndex:assignment.StartIndex+assignment.NumChunks]...) + bundles[result.BlobQuorumInfo.QuorumID].ChunkLen = result.ChunksData.ChunkLen } blobQuorums[blobKey] = append(blobQuorums[blobKey], result.BlobQuorumInfo) diff --git a/disperser/batcher/encoding_streamer_test.go b/disperser/batcher/encoding_streamer_test.go index d84ca03492..5c54dfc880 100644 --- a/disperser/batcher/encoding_streamer_test.go +++ b/disperser/batcher/encoding_streamer_test.go @@ -142,7 +142,7 @@ func TestEncodingQueueLimit(t *testing.T) { } func TestBatchTrigger(t *testing.T) { - encodingStreamer, c := createEncodingStreamer(t, 10, 20_000, streamerConfig) + encodingStreamer, c := createEncodingStreamer(t, 10, 30_000, streamerConfig) blob := makeTestBlob([]*core.SecurityParam{{ QuorumID: 0, @@ -160,7 +160,7 @@ func TestBatchTrigger(t *testing.T) { assert.Nil(t, err) count, size := encodingStreamer.EncodedBlobstore.GetEncodedResultSize() assert.Equal(t, count, 1) - assert.Equal(t, size, uint64(16384)) + assert.Equal(t, size, uint64(26630)) // try encode the same blobs again at different block (this happens when the blob is retried) encodingStreamer.ReferenceBlockNumber = 11 @@ -171,7 +171,7 @@ func TestBatchTrigger(t *testing.T) { count, size = encodingStreamer.EncodedBlobstore.GetEncodedResultSize() assert.Equal(t, count, 1) - assert.Equal(t, size, uint64(16384)) + assert.Equal(t, size, uint64(26630)) // don't notify yet select { @@ -190,7 +190,7 @@ func TestBatchTrigger(t *testing.T) { count, size = encodingStreamer.EncodedBlobstore.GetEncodedResultSize() assert.Equal(t, count, 2) - assert.Equal(t, size, uint64(16384)*2) + assert.Equal(t, size, uint64(26630)*2) // notify select { @@ -246,12 +246,12 @@ func TestStreamingEncoding(t *testing.T) { assert.NotNil(t, encodedResult.Commitment.LengthProof) assert.Greater(t, encodedResult.Commitment.Length, uint(0)) assert.Len(t, encodedResult.Assignments, numOperators) - assert.Len(t, encodedResult.Chunks, 32) + assert.Len(t, encodedResult.ChunksData.Chunks, 32) isRequested = encodingStreamer.EncodedBlobstore.HasEncodingRequested(metadataKey, core.QuorumID(0), 10) assert.True(t, isRequested) count, size = encodingStreamer.EncodedBlobstore.GetEncodedResultSize() assert.Equal(t, count, 1) - assert.Equal(t, size, uint64(16384)) + assert.Equal(t, size, uint64(26630)) // Cancel previous blob so it doesn't get reencoded. err = c.blobStore.MarkBlobFailed(ctx, metadataKey) @@ -281,7 +281,7 @@ func TestStreamingEncoding(t *testing.T) { assert.True(t, isRequested) count, size = encodingStreamer.EncodedBlobstore.GetEncodedResultSize() assert.Equal(t, count, 1) - assert.Equal(t, size, uint64(16384)) + assert.Equal(t, size, uint64(26630)) // Request the same blob, which should be dedupped _, err = c.blobStore.StoreBlob(ctx, &blob, requestedAt) @@ -292,7 +292,7 @@ func TestStreamingEncoding(t *testing.T) { // It should not have been added to the encoded blob store count, size = encodingStreamer.EncodedBlobstore.GetEncodedResultSize() assert.Equal(t, count, 1) - assert.Equal(t, size, uint64(16384)) + assert.Equal(t, size, uint64(26630)) } func TestEncodingFailure(t *testing.T) { @@ -445,7 +445,7 @@ func TestPartialBlob(t *testing.T) { // Check EncodedBlobs assert.Len(t, batch.EncodedBlobs, 1) - assert.Len(t, batch.EncodedBlobs[0].BundlesByOperator, numOperators) + assert.Len(t, batch.EncodedBlobs[0].EncodedBundlesByOperator, numOperators) encodedBlob1 := batch.EncodedBlobs[0] assert.NotNil(t, encodedBlob1) @@ -465,10 +465,10 @@ func TestPartialBlob(t *testing.T) { }}) assert.Contains(t, batch.BlobHeaders, encodedBlob1.BlobHeader) - assert.Len(t, encodedBlob1.BundlesByOperator, numOperators) - for _, bundles := range encodedBlob1.BundlesByOperator { + assert.Len(t, encodedBlob1.EncodedBundlesByOperator, numOperators) + for _, bundles := range encodedBlob1.EncodedBundlesByOperator { assert.Len(t, bundles, 1) - assert.Greater(t, len(bundles[0]), 0) + assert.Greater(t, len(bundles[0].Chunks), 0) break } @@ -674,7 +674,7 @@ func TestGetBatch(t *testing.T) { // Check EncodedBlobs assert.Len(t, batch.EncodedBlobs, 2) - assert.Len(t, batch.EncodedBlobs[0].BundlesByOperator, numOperators) + assert.Len(t, batch.EncodedBlobs[0].EncodedBundlesByOperator, numOperators) var encodedBlob1 core.EncodedBlob var encodedBlob2 core.EncodedBlob @@ -718,10 +718,10 @@ func TestGetBatch(t *testing.T) { }) assert.Contains(t, batch.BlobHeaders, encodedBlob1.BlobHeader) - for _, bundles := range encodedBlob1.BundlesByOperator { + for _, bundles := range encodedBlob1.EncodedBundlesByOperator { assert.Len(t, bundles, 2) - assert.Greater(t, len(bundles[0]), 0) - assert.Greater(t, len(bundles[1]), 0) + assert.Greater(t, len(bundles[0].Chunks), 0) + assert.Greater(t, len(bundles[1].Chunks), 0) break } @@ -739,9 +739,9 @@ func TestGetBatch(t *testing.T) { }, ChunkLength: 8, }}) - for _, bundles := range encodedBlob2.BundlesByOperator { + for _, bundles := range encodedBlob2.EncodedBundlesByOperator { assert.Len(t, bundles, 1) - assert.Greater(t, len(bundles[core.QuorumID(2)]), 0) + assert.Greater(t, len(bundles[core.QuorumID(2)].Chunks), 0) break } assert.Len(t, batch.BlobHeaders, 2) @@ -842,7 +842,7 @@ func TestCreateMinibatch(t *testing.T) { // Check EncodedBlobs assert.Len(t, batch.EncodedBlobs, 2) - assert.Len(t, batch.EncodedBlobs[0].BundlesByOperator, numOperators) + assert.Len(t, batch.EncodedBlobs[0].EncodedBundlesByOperator, numOperators) var encodedBlob1 core.EncodedBlob var encodedBlob2 core.EncodedBlob @@ -886,10 +886,10 @@ func TestCreateMinibatch(t *testing.T) { }) assert.Contains(t, batch.BlobHeaders, encodedBlob1.BlobHeader) - for _, bundles := range encodedBlob1.BundlesByOperator { + for _, bundles := range encodedBlob1.EncodedBundlesByOperator { assert.Len(t, bundles, 2) - assert.Greater(t, len(bundles[0]), 0) - assert.Greater(t, len(bundles[1]), 0) + assert.Greater(t, len(bundles[0].Chunks), 0) + assert.Greater(t, len(bundles[1].Chunks), 0) break } @@ -907,9 +907,9 @@ func TestCreateMinibatch(t *testing.T) { }, ChunkLength: 8, }}) - for _, bundles := range encodedBlob2.BundlesByOperator { + for _, bundles := range encodedBlob2.EncodedBundlesByOperator { assert.Len(t, bundles, 1) - assert.Greater(t, len(bundles[core.QuorumID(2)]), 0) + assert.Greater(t, len(bundles[core.QuorumID(2)].Chunks), 0) break } assert.Len(t, batch.BlobHeaders, 2) diff --git a/disperser/batcher/grpc/dispatcher.go b/disperser/batcher/grpc/dispatcher.go index 1ea3375444..6597492722 100644 --- a/disperser/batcher/grpc/dispatcher.go +++ b/disperser/batcher/grpc/dispatcher.go @@ -52,20 +52,20 @@ func (c *dispatcher) DisperseBatch(ctx context.Context, state *core.IndexedOpera func (c *dispatcher) sendAllChunks(ctx context.Context, state *core.IndexedOperatorState, blobs []core.EncodedBlob, batchHeader *core.BatchHeader, update chan core.SigningMessage) { for id, op := range state.IndexedOperators { go func(op core.IndexedOperatorInfo, id core.OperatorID) { - blobMessages := make([]*core.BlobMessage, 0) + blobMessages := make([]*core.EncodedBlobMessage, 0) hasAnyBundles := false batchHeaderHash, err := batchHeader.GetBatchHeaderHash() if err != nil { return } for _, blob := range blobs { - if _, ok := blob.BundlesByOperator[id]; ok { + if _, ok := blob.EncodedBundlesByOperator[id]; ok { hasAnyBundles = true } - blobMessages = append(blobMessages, &core.BlobMessage{ + blobMessages = append(blobMessages, &core.EncodedBlobMessage{ BlobHeader: blob.BlobHeader, // Bundles will be empty if the operator is not in the quorums blob is dispersed on - Bundles: blob.BundlesByOperator[id], + EncodedBundles: blob.EncodedBundlesByOperator[id], }) } if !hasAnyBundles { @@ -111,7 +111,7 @@ func (c *dispatcher) sendAllChunks(ctx context.Context, state *core.IndexedOpera } } -func (c *dispatcher) sendChunks(ctx context.Context, blobs []*core.BlobMessage, batchHeader *core.BatchHeader, op *core.IndexedOperatorInfo) (*core.Signature, error) { +func (c *dispatcher) sendChunks(ctx context.Context, blobs []*core.EncodedBlobMessage, batchHeader *core.BatchHeader, op *core.IndexedOperatorInfo) (*core.Signature, error) { // TODO Add secure Grpc conn, err := grpc.Dial( @@ -152,7 +152,7 @@ func (c *dispatcher) sendChunks(ctx context.Context, blobs []*core.BlobMessage, // SendBlobsToOperator sends blobs to an operator via the node's StoreBlobs endpoint // It returns the signatures of the blobs sent to the operator in the same order as the blobs // with nil values for blobs that were not attested by the operator -func (c *dispatcher) SendBlobsToOperator(ctx context.Context, blobs []*core.BlobMessage, batchHeader *core.BatchHeader, op *core.IndexedOperatorInfo) ([]*core.Signature, error) { +func (c *dispatcher) SendBlobsToOperator(ctx context.Context, blobs []*core.EncodedBlobMessage, batchHeader *core.BatchHeader, op *core.IndexedOperatorInfo) ([]*core.Signature, error) { // TODO Add secure Grpc conn, err := grpc.Dial( @@ -280,7 +280,7 @@ func (c *dispatcher) SendAttestBatchRequest(ctx context.Context, nodeDispersalCl return &core.Signature{G1Point: point}, nil } -func GetStoreChunksRequest(blobMessages []*core.BlobMessage, batchHeader *core.BatchHeader, useGnarkBundleEncoding bool) (*node.StoreChunksRequest, int64, error) { +func GetStoreChunksRequest(blobMessages []*core.EncodedBlobMessage, batchHeader *core.BatchHeader, useGnarkBundleEncoding bool) (*node.StoreChunksRequest, int64, error) { blobs := make([]*node.Blob, len(blobMessages)) totalSize := int64(0) for i, blob := range blobMessages { @@ -289,7 +289,7 @@ func GetStoreChunksRequest(blobMessages []*core.BlobMessage, batchHeader *core.B if err != nil { return nil, 0, err } - totalSize += int64(blob.Bundles.Size()) + totalSize += getBundlesSize(blob) } request := &node.StoreChunksRequest{ @@ -300,7 +300,7 @@ func GetStoreChunksRequest(blobMessages []*core.BlobMessage, batchHeader *core.B return request, totalSize, nil } -func GetStoreBlobsRequest(blobMessages []*core.BlobMessage, batchHeader *core.BatchHeader, useGnarkBundleEncoding bool) (*node.StoreBlobsRequest, int64, error) { +func GetStoreBlobsRequest(blobMessages []*core.EncodedBlobMessage, batchHeader *core.BatchHeader, useGnarkBundleEncoding bool) (*node.StoreBlobsRequest, int64, error) { blobs := make([]*node.Blob, len(blobMessages)) totalSize := int64(0) for i, blob := range blobMessages { @@ -309,7 +309,7 @@ func GetStoreBlobsRequest(blobMessages []*core.BlobMessage, batchHeader *core.Ba if err != nil { return nil, 0, err } - totalSize += int64(blob.Bundles.Size()) + totalSize += getBundlesSize(blob) } request := &node.StoreBlobsRequest{ @@ -320,7 +320,7 @@ func GetStoreBlobsRequest(blobMessages []*core.BlobMessage, batchHeader *core.Ba return request, totalSize, nil } -func getBlobMessage(blob *core.BlobMessage, useGnarkBundleEncoding bool) (*node.Blob, error) { +func getBlobMessage(blob *core.EncodedBlobMessage, useGnarkBundleEncoding bool) (*node.Blob, error) { if blob.BlobHeader == nil { return nil, errors.New("blob header is nil") } @@ -357,13 +357,20 @@ func getBlobMessage(blob *core.BlobMessage, useGnarkBundleEncoding bool) (*node. } } + var err error bundles := make([]*node.Bundle, len(quorumHeaders)) if useGnarkBundleEncoding { // the ordering of quorums in bundles must be same as in quorumHeaders for i, quorumHeader := range quorumHeaders { quorum := quorumHeader.QuorumId - if bundle, ok := blob.Bundles[uint8(quorum)]; ok { - bundleBytes, err := bundle.Serialize() + if chunksData, ok := blob.EncodedBundles[uint8(quorum)]; ok { + if chunksData.Format != core.GnarkChunkEncodingFormat { + chunksData, err = chunksData.ToGnarkFormat() + if err != nil { + return nil, err + } + } + bundleBytes, err := chunksData.FlattenToBundle() if err != nil { return nil, err } @@ -378,16 +385,18 @@ func getBlobMessage(blob *core.BlobMessage, useGnarkBundleEncoding bool) (*node. } } } else { - data, err := blob.Bundles.Serialize() - if err != nil { - return nil, err - } // the ordering of quorums in bundles must be same as in quorumHeaders for i, quorumHeader := range quorumHeaders { quorum := quorumHeader.QuorumId - if _, ok := blob.Bundles[uint8(quorum)]; ok { + if chunksData, ok := blob.EncodedBundles[uint8(quorum)]; ok { + if chunksData.Format != core.GobChunkEncodingFormat { + chunksData, err = chunksData.ToGobFormat() + if err != nil { + return nil, err + } + } bundles[i] = &node.Bundle{ - Chunks: data[quorum], + Chunks: chunksData.Chunks, } } else { bundles[i] = &node.Bundle{ @@ -417,3 +426,11 @@ func getBatchHeaderMessage(header *core.BatchHeader) *node.BatchHeader { ReferenceBlockNumber: uint32(header.ReferenceBlockNumber), } } + +func getBundlesSize(blob *core.EncodedBlobMessage) int64 { + size := int64(0) + for _, bundle := range blob.EncodedBundles { + size += int64(bundle.Size()) + } + return size +} diff --git a/disperser/batcher/minibatcher.go b/disperser/batcher/minibatcher.go index dfd13419d6..9ae7d1517c 100644 --- a/disperser/batcher/minibatcher.go +++ b/disperser/batcher/minibatcher.go @@ -314,16 +314,16 @@ func (b *Minibatcher) SendBlobsToOperatorWithRetries( opID core.OperatorID, maxNumRetries int, ) ([]*core.Signature, error) { - blobMessages := make([]*core.BlobMessage, 0) + blobMessages := make([]*core.EncodedBlobMessage, 0) hasAnyBundles := false for _, blob := range blobs { - if _, ok := blob.BundlesByOperator[opID]; ok { + if _, ok := blob.EncodedBundlesByOperator[opID]; ok { hasAnyBundles = true } - blobMessages = append(blobMessages, &core.BlobMessage{ + blobMessages = append(blobMessages, &core.EncodedBlobMessage{ BlobHeader: blob.BlobHeader, // Bundles will be empty if the operator is not in the quorums blob is dispersed on - Bundles: blob.BundlesByOperator[opID], + EncodedBundles: blob.EncodedBundlesByOperator[opID], }) } if !hasAnyBundles { diff --git a/disperser/disperser.go b/disperser/disperser.go index cae2980df3..f69f7bfc89 100644 --- a/disperser/disperser.go +++ b/disperser/disperser.go @@ -187,7 +187,7 @@ type BlobStore interface { type Dispatcher interface { DisperseBatch(context.Context, *core.IndexedOperatorState, []core.EncodedBlob, *core.BatchHeader) chan core.SigningMessage - SendBlobsToOperator(ctx context.Context, blobs []*core.BlobMessage, batchHeader *core.BatchHeader, op *core.IndexedOperatorInfo) ([]*core.Signature, error) + SendBlobsToOperator(ctx context.Context, blobs []*core.EncodedBlobMessage, batchHeader *core.BatchHeader, op *core.IndexedOperatorInfo) ([]*core.Signature, error) AttestBatch(ctx context.Context, state *core.IndexedOperatorState, blobHeaderHashes [][32]byte, batchHeader *core.BatchHeader) (chan core.SigningMessage, error) SendAttestBatchRequest(ctx context.Context, nodeDispersalClient node.DispersalClient, blobHeaderHashes [][32]byte, batchHeader *core.BatchHeader, op *core.IndexedOperatorInfo) (*core.Signature, error) } diff --git a/disperser/encoder/client.go b/disperser/encoder/client.go index 6b3858a63b..8a72b08c85 100644 --- a/disperser/encoder/client.go +++ b/disperser/encoder/client.go @@ -5,6 +5,7 @@ import ( "fmt" "time" + "github.com/Layr-Labs/eigenda/core" "github.com/Layr-Labs/eigenda/disperser" pb "github.com/Layr-Labs/eigenda/disperser/api/grpc/encoder" "github.com/Layr-Labs/eigenda/encoding" @@ -24,7 +25,7 @@ func NewEncoderClient(addr string, timeout time.Duration) (disperser.EncoderClie }, nil } -func (c client) EncodeBlob(ctx context.Context, data []byte, encodingParams encoding.EncodingParams) (*encoding.BlobCommitments, []*encoding.Frame, error) { +func (c client) EncodeBlob(ctx context.Context, data []byte, encodingParams encoding.EncodingParams) (*encoding.BlobCommitments, *core.ChunksData, error) { conn, err := grpc.Dial( c.addr, grpc.WithTransportCredentials(insecure.NewCredentials()), @@ -59,18 +60,17 @@ func (c client) EncodeBlob(ctx context.Context, data []byte, encodingParams enco if err != nil { return nil, nil, err } - chunks := make([]*encoding.Frame, len(reply.GetChunks())) - for i, chunk := range reply.GetChunks() { - deserialized, err := new(encoding.Frame).Deserialize(chunk) - if err != nil { - return nil, nil, err - } - chunks[i] = deserialized + chunksData := &core.ChunksData{ + Chunks: reply.GetChunks(), + // TODO(jianoaix): plumb the encoding format for the encoder server. For now it's fine + // as it's hard coded using Gob at Encoder server. + Format: core.GobChunkEncodingFormat, + ChunkLen: int(encodingParams.ChunkLength), } return &encoding.BlobCommitments{ Commitment: commitment, LengthCommitment: lengthCommitment, LengthProof: lengthProof, Length: uint(reply.GetCommitment().GetLength()), - }, chunks, nil + }, chunksData, nil } diff --git a/disperser/encoder_client.go b/disperser/encoder_client.go index 20857af9cd..daffd35136 100644 --- a/disperser/encoder_client.go +++ b/disperser/encoder_client.go @@ -3,9 +3,10 @@ package disperser import ( "context" + "github.com/Layr-Labs/eigenda/core" "github.com/Layr-Labs/eigenda/encoding" ) type EncoderClient interface { - EncodeBlob(ctx context.Context, data []byte, encodingParams encoding.EncodingParams) (*encoding.BlobCommitments, []*encoding.Frame, error) + EncodeBlob(ctx context.Context, data []byte, encodingParams encoding.EncodingParams) (*encoding.BlobCommitments, *core.ChunksData, error) } diff --git a/disperser/local_encoder_client.go b/disperser/local_encoder_client.go index b66cf79dfd..ed55efda0b 100644 --- a/disperser/local_encoder_client.go +++ b/disperser/local_encoder_client.go @@ -4,6 +4,7 @@ import ( "context" "sync" + "github.com/Layr-Labs/eigenda/core" "github.com/Layr-Labs/eigenda/encoding" ) @@ -21,7 +22,7 @@ func NewLocalEncoderClient(prover encoding.Prover) *LocalEncoderClient { } } -func (m *LocalEncoderClient) EncodeBlob(ctx context.Context, data []byte, encodingParams encoding.EncodingParams) (*encoding.BlobCommitments, []*encoding.Frame, error) { +func (m *LocalEncoderClient) EncodeBlob(ctx context.Context, data []byte, encodingParams encoding.EncodingParams) (*encoding.BlobCommitments, *core.ChunksData, error) { m.mu.Lock() defer m.mu.Unlock() commits, chunks, err := m.prover.EncodeAndProve(data, encodingParams) @@ -29,5 +30,19 @@ func (m *LocalEncoderClient) EncodeBlob(ctx context.Context, data []byte, encodi return nil, nil, err } - return &commits, chunks, nil + bytes := make([][]byte, 0, len(chunks)) + for _, c := range chunks { + serialized, err := c.Serialize() + if err != nil { + return nil, nil, err + } + bytes = append(bytes, serialized) + } + chunksData := &core.ChunksData{ + Chunks: bytes, + Format: core.GobChunkEncodingFormat, + ChunkLen: int(encodingParams.ChunkLength), + } + + return &commits, chunksData, nil } diff --git a/disperser/mock/dispatcher.go b/disperser/mock/dispatcher.go index 743de8ebd9..e17dbd0d15 100644 --- a/disperser/mock/dispatcher.go +++ b/disperser/mock/dispatcher.go @@ -66,7 +66,7 @@ func (d *Dispatcher) DisperseBatch(ctx context.Context, state *core.IndexedOpera return update } -func (d *Dispatcher) SendBlobsToOperator(ctx context.Context, blobs []*core.BlobMessage, batchHeader *core.BatchHeader, op *core.IndexedOperatorInfo) ([]*core.Signature, error) { +func (d *Dispatcher) SendBlobsToOperator(ctx context.Context, blobs []*core.EncodedBlobMessage, batchHeader *core.BatchHeader, op *core.IndexedOperatorInfo) ([]*core.Signature, error) { args := d.Called(ctx, blobs, batchHeader, op) if args.Get(0) == nil { return nil, args.Error(1) diff --git a/disperser/mock/encoder.go b/disperser/mock/encoder.go index 0aa6422434..c9d4d1babb 100644 --- a/disperser/mock/encoder.go +++ b/disperser/mock/encoder.go @@ -3,6 +3,7 @@ package mock import ( "context" + "github.com/Layr-Labs/eigenda/core" "github.com/Layr-Labs/eigenda/disperser" "github.com/Layr-Labs/eigenda/encoding" "github.com/stretchr/testify/mock" @@ -18,15 +19,15 @@ func NewMockEncoderClient() *MockEncoderClient { return &MockEncoderClient{} } -func (m *MockEncoderClient) EncodeBlob(ctx context.Context, data []byte, encodingParams encoding.EncodingParams) (*encoding.BlobCommitments, []*encoding.Frame, error) { +func (m *MockEncoderClient) EncodeBlob(ctx context.Context, data []byte, encodingParams encoding.EncodingParams) (*encoding.BlobCommitments, *core.ChunksData, error) { args := m.Called(ctx, data, encodingParams) var commitments *encoding.BlobCommitments if args.Get(0) != nil { commitments = args.Get(0).(*encoding.BlobCommitments) } - var chunks []*encoding.Frame + var chunks *core.ChunksData if args.Get(1) != nil { - chunks = args.Get(1).([]*encoding.Frame) + chunks = args.Get(1).(*core.ChunksData) } return commitments, chunks, args.Error(2) } diff --git a/node/grpc/server_load_test.go b/node/grpc/server_load_test.go index a0ab6bd72d..319da4a174 100644 --- a/node/grpc/server_load_test.go +++ b/node/grpc/server_load_test.go @@ -15,14 +15,14 @@ import ( "github.com/stretchr/testify/assert" ) -func makeBatch(t *testing.T, blobSize int, numBlobs int, advThreshold, quorumThreshold int, refBlockNumber uint) (*core.BatchHeader, map[core.OperatorID][]*core.BlobMessage) { +func makeBatch(t *testing.T, blobSize int, numBlobs int, advThreshold, quorumThreshold int, refBlockNumber uint) (*core.BatchHeader, map[core.OperatorID][]*core.EncodedBlobMessage) { p, _, err := makeTestComponents() assert.NoError(t, err) asn := &core.StdAssignmentCoordinator{} blobHeaders := make([]*core.BlobHeader, numBlobs) blobChunks := make([][]*encoding.Frame, numBlobs) - blobMessagesByOp := make(map[core.OperatorID][]*core.BlobMessage) + blobMessagesByOp := make(map[core.OperatorID][]*core.EncodedBlobMessage) for i := 0; i < numBlobs; i++ { // create data ranData := make([]byte, blobSize) @@ -67,6 +67,13 @@ func makeBatch(t *testing.T, blobSize int, numBlobs int, advThreshold, quorumThr assert.NoError(t, err) blobChunks[i] = chunks + chunkBytes := make([][]byte, len(chunks)) + for _, c := range chunks { + serialized, err := c.Serialize() + assert.NotNil(t, err) + chunkBytes = append(chunkBytes, serialized) + } + // populate blob header blobHeaders[i] = &core.BlobHeader{ BlobCommitments: commits, @@ -75,11 +82,13 @@ func makeBatch(t *testing.T, blobSize int, numBlobs int, advThreshold, quorumThr // populate blob messages for opID, assignment := range quorumInfo.Assignments { - blobMessagesByOp[opID] = append(blobMessagesByOp[opID], &core.BlobMessage{ - BlobHeader: blobHeaders[i], - Bundles: make(core.Bundles), + blobMessagesByOp[opID] = append(blobMessagesByOp[opID], &core.EncodedBlobMessage{ + BlobHeader: blobHeaders[i], + EncodedBundles: make(core.EncodedBundles), }) - blobMessagesByOp[opID][i].Bundles[0] = append(blobMessagesByOp[opID][i].Bundles[0], chunks[assignment.StartIndex:assignment.StartIndex+assignment.NumChunks]...) + blobMessagesByOp[opID][i].EncodedBundles[0].Format = core.GobChunkEncodingFormat + blobMessagesByOp[opID][i].EncodedBundles[0].ChunkLen = int(params.ChunkLength) + blobMessagesByOp[opID][i].EncodedBundles[0].Chunks = append(blobMessagesByOp[opID][i].EncodedBundles[0].Chunks, chunkBytes[assignment.StartIndex:assignment.StartIndex+assignment.NumChunks]...) } } @@ -100,7 +109,7 @@ func TestStoreChunks(t *testing.T) { batchHeader, blobMessagesByOp := makeBatch(t, 200*1024, 50, 80, 100, 1) numTotalChunks := 0 for i := range blobMessagesByOp[opID] { - numTotalChunks += len(blobMessagesByOp[opID][i].Bundles[0]) + numTotalChunks += len(blobMessagesByOp[opID][i].EncodedBundles[0].Chunks) } t.Logf("Batch numTotalChunks: %d", numTotalChunks) req, totalSize, err := dispatcher.GetStoreChunksRequest(blobMessagesByOp[opID], batchHeader, false)