diff --git a/share/blob.go b/share/blob.go index 159bc8d..434f314 100644 --- a/share/blob.go +++ b/share/blob.go @@ -149,3 +149,13 @@ func SortBlobs(blobs []*Blob) { return blobs[i].Compare(blobs[j]) < 0 }) } + +// ToShares converts blob's data back to shares. +func (b *Blob) ToShares() ([]Share, error) { + splitter := NewSparseShareSplitter() + err := splitter.Write(b) + if err != nil { + return nil, err + } + return splitter.Export(), nil +} diff --git a/share/blob_test.go b/share/blob_test.go index 37373e8..771328e 100644 --- a/share/blob_test.go +++ b/share/blob_test.go @@ -63,6 +63,15 @@ func TestBlobConstructor(t *testing.T) { _, err = NewBlob(ns2, data, 0, nil) require.Error(t, err) require.Contains(t, err.Error(), "namespace version must be 0") + + blob, err := NewBlob(ns, data, 0, nil) + require.NoError(t, err) + shares, err := blob.ToShares() + require.NoError(t, err) + blobList, err := parseSparseShares(shares) + require.NoError(t, err) + require.Len(t, blobList, 1) + require.Equal(t, blob, blobList[0]) } func TestNewBlobFromProto(t *testing.T) { diff --git a/share/namespace.go b/share/namespace.go index 7eb6663..c14b71b 100644 --- a/share/namespace.go +++ b/share/namespace.go @@ -2,21 +2,46 @@ package share import ( "bytes" + "encoding/binary" + "encoding/hex" + "encoding/json" + "errors" "fmt" + "slices" ) type Namespace struct { data []byte } +// MarshalJSON encodes namespace to the json encoded bytes. +func (n Namespace) MarshalJSON() ([]byte, error) { + return json.Marshal(n.data) +} + +// UnmarshalJSON decodes json bytes to the namespace. +func (n *Namespace) UnmarshalJSON(data []byte) error { + var buf []byte + if err := json.Unmarshal(data, &buf); err != nil { + return err + } + + ns, err := NewNamespaceFromBytes(buf) + if err != nil { + return err + } + *n = ns + return nil +} + // NewNamespace validates the provided version and id and returns a new namespace. // This should be used for user specified namespaces. func NewNamespace(version uint8, id []byte) (Namespace, error) { - if err := ValidateUserNamespace(version, id); err != nil { + ns := newNamespace(version, id) + if err := ns.validate(); err != nil { return Namespace{}, err } - - return newNamespace(version, id), nil + return ns, nil } func newNamespace(version uint8, id []byte) Namespace { @@ -44,13 +69,12 @@ func NewNamespaceFromBytes(bytes []byte) (Namespace, error) { if len(bytes) != NamespaceSize { return Namespace{}, fmt.Errorf("invalid namespace length: %d. Must be %d bytes", len(bytes), NamespaceSize) } - if err := ValidateUserNamespace(bytes[VersionIndex], bytes[NamespaceVersionSize:]); err != nil { + + ns := Namespace{data: bytes} + if err := ns.validate(); err != nil { return Namespace{}, err } - - return Namespace{ - data: bytes, - }, nil + return ns, nil } // NewV0Namespace returns a new namespace with version 0 and the provided subID. subID @@ -92,35 +116,68 @@ func (n Namespace) ID() []byte { return n.data[NamespaceVersionSize:] } -// ValidateUserNamespace returns an error if the provided version is not +// String stringifies the Namespace. +func (n Namespace) String() string { + return hex.EncodeToString(n.data) +} + +// validate returns an error if the provided version is not // supported or the provided id does not meet the requirements // for the provided version. This should be used for validating // user specified namespaces -func ValidateUserNamespace(version uint8, id []byte) error { - err := validateVersionSupported(version) +func (n Namespace) validate() error { + err := n.validateVersionSupported() if err != nil { return err } - return validateID(version, id) + return n.validateID() +} + +// ValidateForData checks if the Namespace is of real/useful data. +func (n Namespace) ValidateForData() error { + if !n.IsUsableNamespace() { + return fmt.Errorf("invalid data namespace(%s): parity and tail padding namespace are forbidden", n) + } + return nil +} + +// ValidateForBlob verifies whether the Namespace is appropriate for blob data. +// A valid blob namespace must meet two conditions: it cannot be reserved for special purposes, +// and its version must be supported by the system. If either of these conditions is not met, +// an error is returned indicating the issue. This ensures that only valid namespaces are +// used when dealing with blob data. +func (n Namespace) ValidateForBlob() error { + if err := n.ValidateForData(); err != nil { + return err + } + + if n.IsReserved() { + return fmt.Errorf("invalid data namespace(%s): reserved data is forbidden", n) + } + + if !slices.Contains(SupportedBlobNamespaceVersions, n.Version()) { + return fmt.Errorf("blob version %d is not supported", n.Version()) + } + return nil } // validateVersionSupported returns an error if the version is not supported. -func validateVersionSupported(version uint8) error { - if version != NamespaceVersionZero && version != NamespaceVersionMax { - return fmt.Errorf("unsupported namespace version %v", version) +func (n Namespace) validateVersionSupported() error { + if n.Version() != NamespaceVersionZero && n.Version() != NamespaceVersionMax { + return fmt.Errorf("unsupported namespace version %v", n.Version()) } return nil } // validateID returns an error if the provided id does not meet the requirements // for the provided version. -func validateID(version uint8, id []byte) error { - if len(id) != NamespaceIDSize { - return fmt.Errorf("unsupported namespace id length: id %v must be %v bytes but it was %v bytes", id, NamespaceIDSize, len(id)) +func (n Namespace) validateID() error { + if len(n.ID()) != NamespaceIDSize { + return fmt.Errorf("unsupported namespace id length: id %v must be %v bytes but it was %v bytes", n.ID(), NamespaceIDSize, len(n.ID())) } - if version == NamespaceVersionZero && !bytes.HasPrefix(id, NamespaceVersionZeroPrefix) { - return fmt.Errorf("unsupported namespace id with version %v. ID %v must start with %v leading zeros", version, id, len(NamespaceVersionZeroPrefix)) + if n.Version() == NamespaceVersionZero && !bytes.HasPrefix(n.ID(), NamespaceVersionZeroPrefix) { + return fmt.Errorf("unsupported namespace id with version %v. ID %v must start with %v leading zeros", n.Version(), n.ID(), len(NamespaceVersionZeroPrefix)) } return nil } @@ -203,6 +260,52 @@ func (n Namespace) Compare(n2 Namespace) int { return bytes.Compare(n.data, n2.data) } +// AddInt adds arbitrary int value to namespace, treating namespace as big-endian +// implementation of int. It could be helpful for users to create adjacent namespaces. +func (n Namespace) AddInt(val int) (Namespace, error) { + if val == 0 { + return n, nil + } + // Convert the input integer to a byte slice and add it to result slice + result := make([]byte, NamespaceSize) + if val > 0 { + binary.BigEndian.PutUint64(result[NamespaceSize-8:], uint64(val)) + } else { + binary.BigEndian.PutUint64(result[NamespaceSize-8:], uint64(-val)) + } + + // Perform addition byte by byte + var carry int + nn := n.Bytes() + for i := NamespaceSize - 1; i >= 0; i-- { + var sum int + if val > 0 { + sum = int(nn[i]) + int(result[i]) + carry + } else { + sum = int(nn[i]) - int(result[i]) + carry + } + + switch { + case sum > 255: + carry = 1 + sum -= 256 + case sum < 0: + carry = -1 + sum += 256 + default: + carry = 0 + } + + result[i] = uint8(sum) + } + + // Handle any remaining carry + if carry != 0 { + return Namespace{}, errors.New("namespace overflow") + } + return Namespace{data: result}, nil +} + // leftPad returns a new byte slice with the provided byte slice left-padded to the provided size. // If the provided byte slice is already larger than the provided size, the original byte slice is returned. func leftPad(b []byte, size int) []byte { diff --git a/share/namespace_test.go b/share/namespace_test.go index 1fa432a..025e933 100644 --- a/share/namespace_test.go +++ b/share/namespace_test.go @@ -350,6 +350,18 @@ func Test_compareMethods(t *testing.T) { } } +func TestMarshalNamespace(t *testing.T) { + ns := RandomNamespace() + b, err := ns.MarshalJSON() + require.NoError(t, err) + + newNs := Namespace{} + err = newNs.UnmarshalJSON(b) + require.NoError(t, err) + + require.Equal(t, ns, newNs) +} + func BenchmarkEqual(b *testing.B) { n1 := RandomNamespace() n2 := RandomNamespace() diff --git a/share/parse_sparse_shares_test.go b/share/parse_sparse_shares_test.go index 703420e..c69491d 100644 --- a/share/parse_sparse_shares_test.go +++ b/share/parse_sparse_shares_test.go @@ -2,9 +2,7 @@ package share import ( "bytes" - crand "crypto/rand" "fmt" - "math/rand" "testing" "github.com/stretchr/testify/assert" @@ -13,47 +11,63 @@ import ( func Test_parseSparseShares(t *testing.T) { type test struct { - name string - blobSize int - blobCount int + name string + blobSize int + blobCount int + sameNamespace bool } // each test is ran twice, once using blobSize as an exact size, and again // using it as a cap for randomly sized leaves tests := []test{ { - name: "single small blob", - blobSize: 10, - blobCount: 1, + name: "single small blob", + blobSize: 10, + blobCount: 1, + sameNamespace: true, }, { - name: "ten small blobs", - blobSize: 10, - blobCount: 10, + name: "ten small blobs", + blobSize: 10, + blobCount: 10, + sameNamespace: true, }, { - name: "single big blob", - blobSize: ContinuationSparseShareContentSize * 4, - blobCount: 1, + name: "single big blob", + blobSize: ContinuationSparseShareContentSize * 4, + blobCount: 1, + sameNamespace: true, }, { - name: "many big blobs", - blobSize: ContinuationSparseShareContentSize * 4, - blobCount: 10, + name: "many big blobs", + blobSize: ContinuationSparseShareContentSize * 4, + blobCount: 10, + sameNamespace: true, }, { - name: "single exact size blob", - blobSize: FirstSparseShareContentSize, - blobCount: 1, + name: "single exact size blob", + blobSize: FirstSparseShareContentSize, + blobCount: 1, + sameNamespace: true, + }, + { + name: "blobs with different namespaces", + blobSize: FirstSparseShareContentSize, + blobCount: 5, + sameNamespace: false, }, } for _, tc := range tests { // run the tests with identically sized blobs t.Run(fmt.Sprintf("%s identically sized ", tc.name), func(t *testing.T) { - blobs := make([]*Blob, tc.blobCount) - for i := 0; i < tc.blobCount; i++ { - blobs[i] = generateRandomBlob(tc.blobSize) + sizes := make([]int, tc.blobCount) + for i := range sizes { + sizes[i] = tc.blobSize + } + blobs, err := GenerateV0Blobs(sizes, tc.sameNamespace) + if err != nil { + t.Error(err) } SortBlobs(blobs) @@ -70,6 +84,15 @@ func Test_parseSparseShares(t *testing.T) { assert.Equal(t, blobs[i].Namespace(), parsedBlobs[i].Namespace(), "parsed blob namespace does not match") assert.Equal(t, blobs[i].Data(), parsedBlobs[i].Data(), "parsed blob data does not match") } + + if !tc.sameNamespace { + // compare namespaces in case they should not be the same + for i := 0; i < len(blobs); i++ { + for j := i + 1; j < len(blobs); j++ { + require.False(t, parsedBlobs[i].Namespace().Equals(parsedBlobs[j].Namespace())) + } + } + } }) // run the same tests using randomly sized blobs with caps of tc.blobSize @@ -131,42 +154,6 @@ func Test_parseShareVersionOne(t *testing.T) { require.Len(t, parsedBlobs, 1) } -func generateRandomBlobWithNamespace(namespace Namespace, size int) *Blob { - data := make([]byte, size) - _, err := crand.Read(data) - if err != nil { - panic(err) - } - blob, err := NewV0Blob(namespace, data) - if err != nil { - panic(err) - } - return blob -} - -func generateRandomBlob(dataSize int) *Blob { - ns := MustNewV0Namespace(bytes.Repeat([]byte{0x1}, NamespaceVersionZeroIDSize)) - return generateRandomBlobWithNamespace(ns, dataSize) -} - -func generateRandomlySizedBlobs(count, maxBlobSize int) []*Blob { - blobs := make([]*Blob, count) - for i := 0; i < count; i++ { - blobs[i] = generateRandomBlob(rand.Intn(maxBlobSize-1) + 1) - if len(blobs[i].Data()) == 0 { - i-- - } - } - - // this is just to let us use assert.Equal - if count == 0 { - blobs = nil - } - - SortBlobs(blobs) - return blobs -} - func splitBlobs(blobs ...*Blob) ([]Share, error) { writer := NewSparseShareSplitter() for _, blob := range blobs { diff --git a/share/random_blobs.go b/share/random_blobs.go new file mode 100644 index 0000000..aca45ba --- /dev/null +++ b/share/random_blobs.go @@ -0,0 +1,64 @@ +package share + +import ( + "bytes" + crand "crypto/rand" + "math/rand" +) + +// GenerateV0Blobs is a test utility producing v0 share formatted blobs with the +// requested size and namespaces. +func GenerateV0Blobs(sizes []int, sameNamespace bool) ([]*Blob, error) { + blobs := make([]*Blob, 0, len(sizes)) + for _, size := range sizes { + size := rawTxSize(FirstSparseShareContentSize * size) + blob := generateRandomBlob(size) + if !sameNamespace { + ns := RandomBlobNamespace() + var err error + blob, err = NewV0Blob(ns, blob.Data()) + if err != nil { + return nil, err + } + } + + blobs = append(blobs, blob) + } + return blobs, nil +} + +func generateRandomBlobWithNamespace(namespace Namespace, size int) *Blob { + data := make([]byte, size) + _, err := crand.Read(data) + if err != nil { + panic(err) + } + blob, err := NewV0Blob(namespace, data) + if err != nil { + panic(err) + } + return blob +} + +func generateRandomBlob(dataSize int) *Blob { + ns := MustNewV0Namespace(bytes.Repeat([]byte{0x1}, NamespaceVersionZeroIDSize)) + return generateRandomBlobWithNamespace(ns, dataSize) +} + +func generateRandomlySizedBlobs(count, maxBlobSize int) []*Blob { + blobs := make([]*Blob, count) + for i := 0; i < count; i++ { + blobs[i] = generateRandomBlob(rand.Intn(maxBlobSize-1) + 1) + if len(blobs[i].Data()) == 0 { + i-- + } + } + + // this is just to let us use assert.Equal + if count == 0 { + blobs = nil + } + + SortBlobs(blobs) + return blobs +} diff --git a/share/random_namespace.go b/share/random_namespace.go index c13647e..fad86ca 100644 --- a/share/random_namespace.go +++ b/share/random_namespace.go @@ -2,7 +2,6 @@ package share import ( "crypto/rand" - "slices" ) func RandomNamespace() Namespace { @@ -38,22 +37,8 @@ func RandomBlobNamespace() Namespace { for { id := RandomBlobNamespaceID() namespace := MustNewV0Namespace(id) - if isBlobNamespace(namespace) { + if err := namespace.ValidateForBlob(); err == nil { return namespace } } } - -// isBlobNamespace returns an true if this namespace is a valid user-specifiable -// blob namespace. -func isBlobNamespace(ns Namespace) bool { - if ns.IsReserved() { - return false - } - - if !slices.Contains(SupportedBlobNamespaceVersions, ns.Version()) { - return false - } - - return true -} diff --git a/share/random_shares.go b/share/random_shares.go new file mode 100644 index 0000000..2b2913d --- /dev/null +++ b/share/random_shares.go @@ -0,0 +1,70 @@ +package share + +import ( + "bytes" + "crypto/rand" + "fmt" + "sort" +) + +// RandShares generates total amount of shares and fills them with random data. +func RandShares(total int) ([]Share, error) { + if total&(total-1) != 0 { + return nil, fmt.Errorf("total must be power of 2: %d", total) + } + + shares := make([]Share, total) + for i := range shares { + shr := make([]byte, ShareSize) + copy(shr[:NamespaceSize], RandomNamespace().Bytes()) + if _, err := rand.Read(shr[NamespaceSize:]); err != nil { + panic(err) + } + + sh, err := NewShare(shr) + if err != nil { + panic(err) + } + if err = sh.Namespace().ValidateForData(); err != nil { + panic(err) + } + + shares[i] = *sh + } + sort.Slice(shares, func(i, j int) bool { return bytes.Compare(shares[i].ToBytes(), shares[j].ToBytes()) < 0 }) + return shares, nil +} + +// RandSharesWithNamespace is the same as RandShares, but sets the same namespace for all shares. +func RandSharesWithNamespace(namespace Namespace, namespacedAmount, total int) ([]Share, error) { + if total&(total-1) != 0 { + return nil, fmt.Errorf("total must be power of 2: %d", total) + } + + if namespacedAmount > total { + return nil, + fmt.Errorf("namespacedAmount %v must be less than or equal to total: %v", namespacedAmount, total) + } + + shares := make([]Share, total) + for i := range shares { + shr := make([]byte, ShareSize) + if i < namespacedAmount { + copy(shr[:NamespaceSize], namespace.Bytes()) + } else { + copy(shr[:NamespaceSize], RandomNamespace().Bytes()) + } + _, err := rand.Read(shr[NamespaceSize:]) + if err != nil { + panic(err) + } + + sh, err := NewShare(shr) + if err != nil { + panic(err) + } + shares[i] = *sh + } + sort.Slice(shares, func(i, j int) bool { return bytes.Compare(shares[i].ToBytes(), shares[j].ToBytes()) < 0 }) + return shares, nil +} diff --git a/share/share.go b/share/share.go index d8dbd4d..ee91af7 100644 --- a/share/share.go +++ b/share/share.go @@ -9,6 +9,7 @@ package share import ( "bytes" "encoding/binary" + "encoding/json" "fmt" ) @@ -17,6 +18,22 @@ type Share struct { data []byte } +// MarshalJSON encodes share to the json encoded bytes. +func (s Share) MarshalJSON() ([]byte, error) { + return json.Marshal(s.data) +} + +// UnmarshalJSON decodes json bytes to the share. +func (s *Share) UnmarshalJSON(data []byte) error { + var buf []byte + + if err := json.Unmarshal(data, &buf); err != nil { + return err + } + s.data = buf + return validateSize(s.data) +} + // NewShare creates a new share from the raw data, validating it's // size and versioning func NewShare(data []byte) (*Share, error) { diff --git a/share/share_test.go b/share/share_test.go index 2b2af7a..ae96423 100644 --- a/share/share_test.go +++ b/share/share_test.go @@ -228,7 +228,9 @@ func TestUnsupportedShareVersion(t *testing.T) { } func TestShareToBytesAndFromBytes(t *testing.T) { - blobs := []*Blob{generateRandomBlob(580), generateRandomBlob(380), generateRandomBlob(1100)} + blobs, err := GenerateV0Blobs([]int{580, 380, 1100}, true) + require.NoError(t, err) + SortBlobs(blobs) shares, err := splitBlobs(blobs...) require.NoError(t, err) @@ -238,3 +240,16 @@ func TestShareToBytesAndFromBytes(t *testing.T) { require.NoError(t, err) assert.Equal(t, shares, reconstructedShares) } + +func TestMarshalShare(t *testing.T) { + sh, err := RandShares(1) + require.NoError(t, err) + b, err := sh[0].MarshalJSON() + require.NoError(t, err) + + newShare := Share{} + err = newShare.UnmarshalJSON(b) + require.NoError(t, err) + + require.Equal(t, sh[0], newShare) +}