Skip to content

Commit

Permalink
feat: add missing utilities used by celestia-node (#109)
Browse files Browse the repository at this point in the history
Added missing functionality that is commonly used by the node team.

Co-authored-by: Rootul P <[email protected]>
Co-authored-by: Oleg Kovalov <[email protected]>
Co-authored-by: rene <[email protected]>
  • Loading branch information
4 people authored Oct 17, 2024
1 parent 9ff313b commit f47b641
Show file tree
Hide file tree
Showing 10 changed files with 368 additions and 96 deletions.
10 changes: 10 additions & 0 deletions share/blob.go
Original file line number Diff line number Diff line change
Expand Up @@ -149,3 +149,13 @@ func SortBlobs(blobs []*Blob) {
return blobs[i].Compare(blobs[j]) < 0
})
}

// ToShares converts blob's data back to shares.
func (b *Blob) ToShares() ([]Share, error) {
splitter := NewSparseShareSplitter()
err := splitter.Write(b)
if err != nil {
return nil, err
}
return splitter.Export(), nil
}
9 changes: 9 additions & 0 deletions share/blob_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,15 @@ func TestBlobConstructor(t *testing.T) {
_, err = NewBlob(ns2, data, 0, nil)
require.Error(t, err)
require.Contains(t, err.Error(), "namespace version must be 0")

blob, err := NewBlob(ns, data, 0, nil)
require.NoError(t, err)
shares, err := blob.ToShares()
require.NoError(t, err)
blobList, err := parseSparseShares(shares)
require.NoError(t, err)
require.Len(t, blobList, 1)
require.Equal(t, blob, blobList[0])
}

func TestNewBlobFromProto(t *testing.T) {
Expand Down
143 changes: 123 additions & 20 deletions share/namespace.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,21 +2,46 @@ package share

import (
"bytes"
"encoding/binary"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"slices"
)

type Namespace struct {
data []byte
}

// MarshalJSON encodes namespace to the json encoded bytes.
func (n Namespace) MarshalJSON() ([]byte, error) {
return json.Marshal(n.data)
}

// UnmarshalJSON decodes json bytes to the namespace.
func (n *Namespace) UnmarshalJSON(data []byte) error {
var buf []byte
if err := json.Unmarshal(data, &buf); err != nil {
return err
}

ns, err := NewNamespaceFromBytes(buf)
if err != nil {
return err
}
*n = ns
return nil
}

// NewNamespace validates the provided version and id and returns a new namespace.
// This should be used for user specified namespaces.
func NewNamespace(version uint8, id []byte) (Namespace, error) {
if err := ValidateUserNamespace(version, id); err != nil {
ns := newNamespace(version, id)
if err := ns.validate(); err != nil {
return Namespace{}, err
}

return newNamespace(version, id), nil
return ns, nil
}

func newNamespace(version uint8, id []byte) Namespace {
Expand Down Expand Up @@ -44,13 +69,12 @@ func NewNamespaceFromBytes(bytes []byte) (Namespace, error) {
if len(bytes) != NamespaceSize {
return Namespace{}, fmt.Errorf("invalid namespace length: %d. Must be %d bytes", len(bytes), NamespaceSize)
}
if err := ValidateUserNamespace(bytes[VersionIndex], bytes[NamespaceVersionSize:]); err != nil {

ns := Namespace{data: bytes}
if err := ns.validate(); err != nil {
return Namespace{}, err
}

return Namespace{
data: bytes,
}, nil
return ns, nil
}

// NewV0Namespace returns a new namespace with version 0 and the provided subID. subID
Expand Down Expand Up @@ -92,35 +116,68 @@ func (n Namespace) ID() []byte {
return n.data[NamespaceVersionSize:]
}

// ValidateUserNamespace returns an error if the provided version is not
// String stringifies the Namespace.
func (n Namespace) String() string {
return hex.EncodeToString(n.data)
}

// validate returns an error if the provided version is not
// supported or the provided id does not meet the requirements
// for the provided version. This should be used for validating
// user specified namespaces
func ValidateUserNamespace(version uint8, id []byte) error {
err := validateVersionSupported(version)
func (n Namespace) validate() error {
err := n.validateVersionSupported()
if err != nil {
return err
}
return validateID(version, id)
return n.validateID()
}

// ValidateForData checks if the Namespace is of real/useful data.
func (n Namespace) ValidateForData() error {
if !n.IsUsableNamespace() {
return fmt.Errorf("invalid data namespace(%s): parity and tail padding namespace are forbidden", n)
}
return nil
}

// ValidateForBlob verifies whether the Namespace is appropriate for blob data.
// A valid blob namespace must meet two conditions: it cannot be reserved for special purposes,
// and its version must be supported by the system. If either of these conditions is not met,
// an error is returned indicating the issue. This ensures that only valid namespaces are
// used when dealing with blob data.
func (n Namespace) ValidateForBlob() error {
if err := n.ValidateForData(); err != nil {
return err
}

if n.IsReserved() {
return fmt.Errorf("invalid data namespace(%s): reserved data is forbidden", n)
}

if !slices.Contains(SupportedBlobNamespaceVersions, n.Version()) {
return fmt.Errorf("blob version %d is not supported", n.Version())
}
return nil
}

// validateVersionSupported returns an error if the version is not supported.
func validateVersionSupported(version uint8) error {
if version != NamespaceVersionZero && version != NamespaceVersionMax {
return fmt.Errorf("unsupported namespace version %v", version)
func (n Namespace) validateVersionSupported() error {
if n.Version() != NamespaceVersionZero && n.Version() != NamespaceVersionMax {
return fmt.Errorf("unsupported namespace version %v", n.Version())
}
return nil
}

// validateID returns an error if the provided id does not meet the requirements
// for the provided version.
func validateID(version uint8, id []byte) error {
if len(id) != NamespaceIDSize {
return fmt.Errorf("unsupported namespace id length: id %v must be %v bytes but it was %v bytes", id, NamespaceIDSize, len(id))
func (n Namespace) validateID() error {
if len(n.ID()) != NamespaceIDSize {
return fmt.Errorf("unsupported namespace id length: id %v must be %v bytes but it was %v bytes", n.ID(), NamespaceIDSize, len(n.ID()))
}

if version == NamespaceVersionZero && !bytes.HasPrefix(id, NamespaceVersionZeroPrefix) {
return fmt.Errorf("unsupported namespace id with version %v. ID %v must start with %v leading zeros", version, id, len(NamespaceVersionZeroPrefix))
if n.Version() == NamespaceVersionZero && !bytes.HasPrefix(n.ID(), NamespaceVersionZeroPrefix) {
return fmt.Errorf("unsupported namespace id with version %v. ID %v must start with %v leading zeros", n.Version(), n.ID(), len(NamespaceVersionZeroPrefix))
}
return nil
}
Expand Down Expand Up @@ -203,6 +260,52 @@ func (n Namespace) Compare(n2 Namespace) int {
return bytes.Compare(n.data, n2.data)
}

// AddInt adds arbitrary int value to namespace, treating namespace as big-endian
// implementation of int. It could be helpful for users to create adjacent namespaces.
func (n Namespace) AddInt(val int) (Namespace, error) {
if val == 0 {
return n, nil
}
// Convert the input integer to a byte slice and add it to result slice
result := make([]byte, NamespaceSize)
if val > 0 {
binary.BigEndian.PutUint64(result[NamespaceSize-8:], uint64(val))
} else {
binary.BigEndian.PutUint64(result[NamespaceSize-8:], uint64(-val))
}

// Perform addition byte by byte
var carry int
nn := n.Bytes()
for i := NamespaceSize - 1; i >= 0; i-- {
var sum int
if val > 0 {
sum = int(nn[i]) + int(result[i]) + carry
} else {
sum = int(nn[i]) - int(result[i]) + carry
}

switch {
case sum > 255:
carry = 1
sum -= 256
case sum < 0:
carry = -1
sum += 256
default:
carry = 0
}

result[i] = uint8(sum)
}

// Handle any remaining carry
if carry != 0 {
return Namespace{}, errors.New("namespace overflow")
}
return Namespace{data: result}, nil
}

// leftPad returns a new byte slice with the provided byte slice left-padded to the provided size.
// If the provided byte slice is already larger than the provided size, the original byte slice is returned.
func leftPad(b []byte, size int) []byte {
Expand Down
12 changes: 12 additions & 0 deletions share/namespace_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -350,6 +350,18 @@ func Test_compareMethods(t *testing.T) {
}
}

func TestMarshalNamespace(t *testing.T) {
ns := RandomNamespace()
b, err := ns.MarshalJSON()
require.NoError(t, err)

newNs := Namespace{}
err = newNs.UnmarshalJSON(b)
require.NoError(t, err)

require.Equal(t, ns, newNs)
}

func BenchmarkEqual(b *testing.B) {
n1 := RandomNamespace()
n2 := RandomNamespace()
Expand Down
Loading

0 comments on commit f47b641

Please sign in to comment.