diff --git a/cmd/decode-state-values/main.go b/cmd/decode-state-values/main.go index 5c61e11ba0..91f3a6d840 100644 --- a/cmd/decode-state-values/main.go +++ b/cmd/decode-state-values/main.go @@ -234,8 +234,13 @@ type interpreterStorage struct { var _ interpreter.Storage = &interpreterStorage{} -func (i interpreterStorage) GetStorageMap(_ *interpreter.Interpreter, _ common.Address, _ common.StorageDomain, _ bool) *interpreter.DomainStorageMap { - panic("unexpected GetStorageMap call") +func (i interpreterStorage) GetDomainStorageMap( + _ *interpreter.Interpreter, + _ common.Address, + _ common.StorageDomain, + _ bool, +) *interpreter.DomainStorageMap { + panic("unexpected GetDomainStorageMap call") } func (i interpreterStorage) CheckHealth() error { diff --git a/common/address.go b/common/address.go index 5c1a354f59..04c2bade70 100644 --- a/common/address.go +++ b/common/address.go @@ -19,6 +19,7 @@ package common import ( + "bytes" "encoding/hex" goErrors "errors" "fmt" @@ -112,6 +113,10 @@ func (a Address) HexWithPrefix() string { return fmt.Sprintf("0x%x", [AddressLength]byte(a)) } +func (a Address) Compare(other Address) int { + return bytes.Compare(a[:], other[:]) +} + // HexToAddress converts a hex string to an Address after // ensuring that the hex string starts with the prefix 0x. func HexToAddressAssertPrefix(h string) (Address, error) { diff --git a/interpreter/account_storagemap.go b/interpreter/account_storagemap.go index d06aaafe87..25e1d881a1 100644 --- a/interpreter/account_storagemap.go +++ b/interpreter/account_storagemap.go @@ -164,12 +164,12 @@ func (s *AccountStorageMap) NewDomain( func (s *AccountStorageMap) WriteDomain( interpreter *Interpreter, domain common.StorageDomain, - storageMap *DomainStorageMap, + domainStorageMap *DomainStorageMap, ) (existed bool) { - if storageMap == nil { + if domainStorageMap == nil { return s.removeDomain(interpreter, domain) } - return s.setDomain(interpreter, domain, storageMap) + return s.setDomain(interpreter, domain, domainStorageMap) } // setDomain sets domain storage map in the account storage map and returns true if domain previously existed. @@ -177,7 +177,7 @@ func (s *AccountStorageMap) WriteDomain( func (s *AccountStorageMap) setDomain( interpreter *Interpreter, domain common.StorageDomain, - storageMap *DomainStorageMap, + newDomainStorageMap *DomainStorageMap, ) (existed bool) { interpreter.recordStorageMutation() @@ -187,7 +187,7 @@ func (s *AccountStorageMap) setDomain( key.AtreeValueCompare, key.AtreeValueHashInput, key.AtreeValue(), - storageMap.orderedMap, + newDomainStorageMap.orderedMap, ) if err != nil { panic(errors.NewExternalError(err)) @@ -196,10 +196,10 @@ func (s *AccountStorageMap) setDomain( existed = existingValueStorable != nil if existed { // Create domain storage map from overwritten storable - domainStorageMap := newDomainStorageMapWithAtreeStorable(s.orderedMap.Storage, existingValueStorable) + existingDomainStorageMap := newDomainStorageMapWithAtreeStorable(s.orderedMap.Storage, existingValueStorable) // Deep remove elements in domain storage map - domainStorageMap.DeepRemove(interpreter, true) + existingDomainStorageMap.DeepRemove(interpreter, true) // Remove domain storage map slab interpreter.RemoveReferencedSlab(existingValueStorable) diff --git a/interpreter/account_storagemap_test.go b/interpreter/account_storagemap_test.go index 6a8794a60e..a2939e14ac 100644 --- a/interpreter/account_storagemap_test.go +++ b/interpreter/account_storagemap_test.go @@ -43,8 +43,16 @@ func TestAccountStorageMapDomainExists(t *testing.T) { address := common.MustBytesToAddress([]byte{0x1}) t.Run("empty", func(t *testing.T) { + t.Parallel() + ledger := NewTestLedger(nil, nil) - storage := runtime.NewStorage(ledger, nil) + storage := runtime.NewStorage( + ledger, + nil, + runtime.StorageConfig{ + StorageFormatV2Enabled: true, + }, + ) accountStorageMap := interpreter.NewAccountStorageMap(nil, storage, atree.Address(address)) require.NotNil(t, accountStorageMap) @@ -59,10 +67,18 @@ func TestAccountStorageMapDomainExists(t *testing.T) { }) t.Run("non-empty", func(t *testing.T) { + t.Parallel() + random := rand.New(rand.NewSource(42)) ledger := NewTestLedger(nil, nil) - storage := runtime.NewStorage(ledger, nil) + storage := runtime.NewStorage( + ledger, + nil, + runtime.StorageConfig{ + StorageFormatV2Enabled: true, + }, + ) // Turn off AtreeStorageValidationEnabled and explicitly check atree storage health at the end of test. // This is because AccountStorageMap isn't created through runtime.Storage, so there isn't any @@ -97,8 +113,16 @@ func TestAccountStorageMapGetDomain(t *testing.T) { address := common.MustBytesToAddress([]byte{0x1}) t.Run("empty", func(t *testing.T) { + t.Parallel() + ledger := NewTestLedger(nil, nil) - storage := runtime.NewStorage(ledger, nil) + storage := runtime.NewStorage( + ledger, + nil, + runtime.StorageConfig{ + StorageFormatV2Enabled: true, + }, + ) // Turn off AtreeStorageValidationEnabled and explicitly check atree storage health at the end of test. // This is because AccountStorageMap isn't created through runtime.Storage, so there isn't any @@ -118,18 +142,26 @@ func TestAccountStorageMapGetDomain(t *testing.T) { for _, domain := range common.AllStorageDomains { const createIfNotExists = false - storagemap := accountStorageMap.GetDomain(nil, inter, domain, createIfNotExists) - require.Nil(t, storagemap) + domainStorageMap := accountStorageMap.GetDomain(nil, inter, domain, createIfNotExists) + require.Nil(t, domainStorageMap) } CheckAtreeStorageHealth(t, storage, []atree.SlabID{accountStorageMap.SlabID()}) }) t.Run("non-empty", func(t *testing.T) { + t.Parallel() + random := rand.New(rand.NewSource(42)) ledger := NewTestLedger(nil, nil) - storage := runtime.NewStorage(ledger, nil) + storage := runtime.NewStorage( + ledger, + nil, + runtime.StorageConfig{ + StorageFormatV2Enabled: true, + }, + ) // Turn off AtreeStorageValidationEnabled and explicitly check atree storage health at the end of test. // This is because AccountStorageMap isn't created through runtime.Storage, so there isn't any @@ -150,11 +182,11 @@ func TestAccountStorageMapGetDomain(t *testing.T) { for _, domain := range common.AllStorageDomains { const createIfNotExists = false - domainStoragemap := accountStorageMap.GetDomain(nil, inter, domain, createIfNotExists) - require.Equal(t, slices.Contains(existingDomains, domain), domainStoragemap != nil) + domainStorageMap := accountStorageMap.GetDomain(nil, inter, domain, createIfNotExists) + require.Equal(t, slices.Contains(existingDomains, domain), domainStorageMap != nil) - if domainStoragemap != nil { - checkDomainStorageMapData(t, inter, domainStoragemap, accountValues[domain]) + if domainStorageMap != nil { + checkDomainStorageMapData(t, inter, domainStorageMap, accountValues[domain]) } } @@ -168,8 +200,16 @@ func TestAccountStorageMapCreateDomain(t *testing.T) { address := common.MustBytesToAddress([]byte{0x1}) t.Run("empty", func(t *testing.T) { + t.Parallel() + ledger := NewTestLedger(nil, nil) - storage := runtime.NewStorage(ledger, nil) + storage := runtime.NewStorage( + ledger, + nil, + runtime.StorageConfig{ + StorageFormatV2Enabled: true, + }, + ) // Turn off AtreeStorageValidationEnabled and explicitly check atree storage health at the end of test. // This is because AccountStorageMap isn't created through runtime.Storage, so there isn't any @@ -193,9 +233,9 @@ func TestAccountStorageMapCreateDomain(t *testing.T) { for _, domain := range common.AllStorageDomains { const createIfNotExists = true - domainStoragemap := accountStorageMap.GetDomain(nil, inter, domain, createIfNotExists) - require.NotNil(t, domainStoragemap) - require.Equal(t, uint64(0), domainStoragemap.Count()) + domainStorageMap := accountStorageMap.GetDomain(nil, inter, domain, createIfNotExists) + require.NotNil(t, domainStorageMap) + require.Equal(t, uint64(0), domainStorageMap.Count()) accountValues[domain] = make(domainStorageMapValues) } @@ -211,10 +251,18 @@ func TestAccountStorageMapCreateDomain(t *testing.T) { }) t.Run("non-empty", func(t *testing.T) { + t.Parallel() + random := rand.New(rand.NewSource(42)) ledger := NewTestLedger(nil, nil) - storage := runtime.NewStorage(ledger, nil) + storage := runtime.NewStorage( + ledger, + nil, + runtime.StorageConfig{ + StorageFormatV2Enabled: true, + }, + ) // Turn off AtreeStorageValidationEnabled and explicitly check atree storage health at the end of test. // This is because AccountStorageMap isn't created through runtime.Storage, so there isn't any @@ -237,9 +285,9 @@ func TestAccountStorageMapCreateDomain(t *testing.T) { for _, domain := range common.AllStorageDomains { const createIfNotExists = true - domainStoragemap := accountStorageMap.GetDomain(nil, inter, domain, createIfNotExists) - require.NotNil(t, domainStoragemap) - require.Equal(t, uint64(len(accountValues[domain])), domainStoragemap.Count()) + domainStorageMap := accountStorageMap.GetDomain(nil, inter, domain, createIfNotExists) + require.NotNil(t, domainStorageMap) + require.Equal(t, uint64(len(accountValues[domain])), domainStorageMap.Count()) if !slices.Contains(existingDomains, domain) { accountValues[domain] = make(domainStorageMapValues) @@ -263,10 +311,18 @@ func TestAccountStorageMapSetAndUpdateDomain(t *testing.T) { address := common.MustBytesToAddress([]byte{0x1}) t.Run("empty", func(t *testing.T) { + t.Parallel() + random := rand.New(rand.NewSource(42)) ledger := NewTestLedger(nil, nil) - storage := runtime.NewStorage(ledger, nil) + storage := runtime.NewStorage( + ledger, + nil, + runtime.StorageConfig{ + StorageFormatV2Enabled: true, + }, + ) // Turn off AtreeStorageValidationEnabled and explicitly check atree storage health at the end of test. // This is because AccountStorageMap isn't created through runtime.Storage, so there isn't any @@ -311,10 +367,18 @@ func TestAccountStorageMapSetAndUpdateDomain(t *testing.T) { }) t.Run("non-empty", func(t *testing.T) { + t.Parallel() + random := rand.New(rand.NewSource(42)) ledger := NewTestLedger(nil, nil) - storage := runtime.NewStorage(ledger, nil) + storage := runtime.NewStorage( + ledger, + nil, + runtime.StorageConfig{ + StorageFormatV2Enabled: true, + }, + ) // Turn off AtreeStorageValidationEnabled and explicitly check atree storage health at the end of test. // This is because AccountStorageMap isn't created through runtime.Storage, so there isn't any @@ -363,8 +427,16 @@ func TestAccountStorageMapRemoveDomain(t *testing.T) { address := common.MustBytesToAddress([]byte{0x1}) t.Run("empty", func(t *testing.T) { + t.Parallel() + ledger := NewTestLedger(nil, nil) - storage := runtime.NewStorage(ledger, nil) + storage := runtime.NewStorage( + ledger, + nil, + runtime.StorageConfig{ + StorageFormatV2Enabled: true, + }, + ) // Turn off AtreeStorageValidationEnabled and explicitly check atree storage health at the end of test. // This is because AccountStorageMap isn't created through runtime.Storage, so there isn't any @@ -402,10 +474,18 @@ func TestAccountStorageMapRemoveDomain(t *testing.T) { }) t.Run("non-empty", func(t *testing.T) { + t.Parallel() + random := rand.New(rand.NewSource(42)) ledger := NewTestLedger(nil, nil) - storage := runtime.NewStorage(ledger, nil) + storage := runtime.NewStorage( + ledger, + nil, + runtime.StorageConfig{ + StorageFormatV2Enabled: true, + }, + ) // Turn off AtreeStorageValidationEnabled and explicitly check atree storage health at the end of test. // This is because AccountStorageMap isn't created through runtime.Storage, so there isn't any @@ -451,8 +531,16 @@ func TestAccountStorageMapIterator(t *testing.T) { address := common.MustBytesToAddress([]byte{0x1}) t.Run("empty", func(t *testing.T) { + t.Parallel() + ledger := NewTestLedger(nil, nil) - storage := runtime.NewStorage(ledger, nil) + storage := runtime.NewStorage( + ledger, + nil, + runtime.StorageConfig{ + StorageFormatV2Enabled: true, + }, + ) // Turn off AtreeStorageValidationEnabled and explicitly check atree storage health at the end of test. // This is because AccountStorageMap isn't created through runtime.Storage, so there isn't any @@ -487,10 +575,18 @@ func TestAccountStorageMapIterator(t *testing.T) { }) t.Run("non-empty", func(t *testing.T) { + t.Parallel() + random := rand.New(rand.NewSource(42)) ledger := NewTestLedger(nil, nil) - storage := runtime.NewStorage(ledger, nil) + storage := runtime.NewStorage( + ledger, + nil, + runtime.StorageConfig{ + StorageFormatV2Enabled: true, + }, + ) // Turn off AtreeStorageValidationEnabled and explicitly check atree storage health at the end of test. // This is because AccountStorageMap isn't created through runtime.Storage, so there isn't any @@ -548,8 +644,16 @@ func TestAccountStorageMapDomains(t *testing.T) { address := common.MustBytesToAddress([]byte{0x1}) t.Run("empty", func(t *testing.T) { + t.Parallel() + ledger := NewTestLedger(nil, nil) - storage := runtime.NewStorage(ledger, nil) + storage := runtime.NewStorage( + ledger, + nil, + runtime.StorageConfig{ + StorageFormatV2Enabled: true, + }, + ) accountStorageMap := interpreter.NewAccountStorageMap(nil, storage, atree.Address(address)) require.NotNil(t, accountStorageMap) @@ -562,10 +666,18 @@ func TestAccountStorageMapDomains(t *testing.T) { }) t.Run("non-empty", func(t *testing.T) { + t.Parallel() + random := rand.New(rand.NewSource(42)) ledger := NewTestLedger(nil, nil) - storage := runtime.NewStorage(ledger, nil) + storage := runtime.NewStorage( + ledger, + nil, + runtime.StorageConfig{ + StorageFormatV2Enabled: true, + }, + ) // Turn off automatic AtreeStorageValidationEnabled and explicitly check atree storage health directly. // This is because AccountStorageMap isn't created through storage, so there isn't any account register to match AccountStorageMap root slab. @@ -600,9 +712,17 @@ func TestAccountStorageMapLoadFromRootSlabID(t *testing.T) { address := common.MustBytesToAddress([]byte{0x1}) t.Run("empty", func(t *testing.T) { + t.Parallel() + init := func() (atree.SlabID, accountStorageMapValues, map[string][]byte, map[string]uint64) { ledger := NewTestLedger(nil, nil) - storage := runtime.NewStorage(ledger, nil) + storage := runtime.NewStorage( + ledger, + nil, + runtime.StorageConfig{ + StorageFormatV2Enabled: true, + }, + ) inter := NewTestInterpreterWithStorage(t, storage) @@ -632,7 +752,13 @@ func TestAccountStorageMapLoadFromRootSlabID(t *testing.T) { random := rand.New(rand.NewSource(42)) ledger := NewTestLedger(nil, nil) - storage := runtime.NewStorage(ledger, nil) + storage := runtime.NewStorage( + ledger, + nil, + runtime.StorageConfig{ + StorageFormatV2Enabled: true, + }, + ) // Turn off automatic AtreeStorageValidationEnabled and explicitly check atree storage health directly. // This is because AccountStorageMap isn't created through storage, so there isn't any account register to match AccountStorageMap root slab. @@ -728,7 +854,13 @@ func checkAccountStorageMapDataWithRawData( ) { // Create new storage from raw data ledger := NewTestLedgerWithData(nil, nil, storedValues, storageIndices) - storage := runtime.NewStorage(ledger, nil) + storage := runtime.NewStorage( + ledger, + nil, + runtime.StorageConfig{ + StorageFormatV2Enabled: true, + }, + ) inter := NewTestInterpreterWithStorage(tb, storage) diff --git a/interpreter/account_test.go b/interpreter/account_test.go index 595d0d7352..8f1f9c4214 100644 --- a/interpreter/account_test.go +++ b/interpreter/account_test.go @@ -483,7 +483,7 @@ func testAccountWithErrorHandler( getAccountValues := func() map[storageKey]interpreter.Value { accountValues := make(map[storageKey]interpreter.Value) - for storageMapKey, accountStorage := range inter.Storage().(interpreter.InMemoryStorage).StorageMaps { + for storageMapKey, accountStorage := range inter.Storage().(interpreter.InMemoryStorage).DomainStorageMaps { iterator := accountStorage.Iterator(inter) for { key, value := iterator.Next() diff --git a/interpreter/domain_storagemap.go b/interpreter/domain_storagemap.go index 336828ce64..e8f79bca35 100644 --- a/interpreter/domain_storagemap.go +++ b/interpreter/domain_storagemap.go @@ -110,7 +110,7 @@ func NewDomainStorageMapWithAtreeValue(value atree.Value) *DomainStorageMap { } // ValueExists returns true if the given key exists in the storage map. -func (s DomainStorageMap) ValueExists(key StorageMapKey) bool { +func (s *DomainStorageMap) ValueExists(key StorageMapKey) bool { exists, err := s.orderedMap.Has( key.AtreeValueCompare, key.AtreeValueHashInput, @@ -125,7 +125,7 @@ func (s DomainStorageMap) ValueExists(key StorageMapKey) bool { // ReadValue returns the value for the given key. // Returns nil if the key does not exist. -func (s DomainStorageMap) ReadValue(gauge common.MemoryGauge, key StorageMapKey) Value { +func (s *DomainStorageMap) ReadValue(gauge common.MemoryGauge, key StorageMapKey) Value { storedValue, err := s.orderedMap.Get( key.AtreeValueCompare, key.AtreeValueHashInput, @@ -146,7 +146,7 @@ func (s DomainStorageMap) ReadValue(gauge common.MemoryGauge, key StorageMapKey) // If the given value is nil, the key is removed. // If the given value is non-nil, the key is added/updated. // Returns true if a value previously existed at the given key. -func (s DomainStorageMap) WriteValue(interpreter *Interpreter, key StorageMapKey, value atree.Value) (existed bool) { +func (s *DomainStorageMap) WriteValue(interpreter *Interpreter, key StorageMapKey, value atree.Value) (existed bool) { if value == nil { return s.RemoveValue(interpreter, key) } else { @@ -157,7 +157,7 @@ func (s DomainStorageMap) WriteValue(interpreter *Interpreter, key StorageMapKey // SetValue sets a value in the storage map. // If the given key already stores a value, it is overwritten. // Returns true if given key already exists and existing value is overwritten. -func (s DomainStorageMap) SetValue(interpreter *Interpreter, key StorageMapKey, value atree.Value) (existed bool) { +func (s *DomainStorageMap) SetValue(interpreter *Interpreter, key StorageMapKey, value atree.Value) (existed bool) { interpreter.recordStorageMutation() existingStorable, err := s.orderedMap.Set( @@ -184,7 +184,7 @@ func (s DomainStorageMap) SetValue(interpreter *Interpreter, key StorageMapKey, } // RemoveValue removes a value in the storage map, if it exists. -func (s DomainStorageMap) RemoveValue(interpreter *Interpreter, key StorageMapKey) (existed bool) { +func (s *DomainStorageMap) RemoveValue(interpreter *Interpreter, key StorageMapKey) (existed bool) { interpreter.recordStorageMutation() existingKeyStorable, existingValueStorable, err := s.orderedMap.Remove( @@ -270,22 +270,26 @@ func (s *DomainStorageMap) DeepRemove(interpreter *Interpreter, hasNoParentConta } } -func (s DomainStorageMap) ValueID() atree.ValueID { +func (s *DomainStorageMap) SlabID() atree.SlabID { + return s.orderedMap.SlabID() +} + +func (s *DomainStorageMap) ValueID() atree.ValueID { return s.orderedMap.ValueID() } -func (s DomainStorageMap) Count() uint64 { +func (s *DomainStorageMap) Count() uint64 { return s.orderedMap.Count() } -func (s DomainStorageMap) Inlined() bool { +func (s *DomainStorageMap) Inlined() bool { // This is only used for testing currently. return s.orderedMap.Inlined() } // Iterator returns an iterator (StorageMapIterator), // which allows iterating over the keys and values of the storage map -func (s DomainStorageMap) Iterator(gauge common.MemoryGauge) DomainStorageMapIterator { +func (s *DomainStorageMap) Iterator(gauge common.MemoryGauge) DomainStorageMapIterator { mapIterator, err := s.orderedMap.Iterator( StorageMapKeyAtreeValueComparator, StorageMapKeyAtreeValueHashInput, diff --git a/interpreter/domain_storagemap_test.go b/interpreter/domain_storagemap_test.go index 9b1985dce1..5eb2db5c66 100644 --- a/interpreter/domain_storagemap_test.go +++ b/interpreter/domain_storagemap_test.go @@ -40,8 +40,14 @@ func TestDomainStorageMapValueExists(t *testing.T) { address := common.MustBytesToAddress([]byte{0x1}) t.Run("empty", func(t *testing.T) { + t.Parallel() + ledger := NewTestLedger(nil, nil) - storage := runtime.NewStorage(ledger, nil) + storage := runtime.NewStorage( + ledger, + nil, + runtime.StorageConfig{}, + ) domainStorageMap := interpreter.NewDomainStorageMap(nil, storage, atree.Address(address)) require.NotNil(t, domainStorageMap) @@ -56,10 +62,16 @@ func TestDomainStorageMapValueExists(t *testing.T) { }) t.Run("non-empty", func(t *testing.T) { + t.Parallel() + random := rand.New(rand.NewSource(42)) ledger := NewTestLedger(nil, nil) - storage := runtime.NewStorage(ledger, nil) + storage := runtime.NewStorage( + ledger, + nil, + runtime.StorageConfig{}, + ) // Turn off AtreeStorageValidationEnabled and explicitly check atree storage health at the end of test. // This is because DomainStorageMap isn't created through runtime.Storage, so there isn't any @@ -103,8 +115,14 @@ func TestDomainStorageMapReadValue(t *testing.T) { address := common.MustBytesToAddress([]byte{0x1}) t.Run("empty", func(t *testing.T) { + t.Parallel() + ledger := NewTestLedger(nil, nil) - storage := runtime.NewStorage(ledger, nil) + storage := runtime.NewStorage( + ledger, + nil, + runtime.StorageConfig{}, + ) domainStorageMap := interpreter.NewDomainStorageMap(nil, storage, atree.Address(address)) require.NotNil(t, domainStorageMap) @@ -119,10 +137,16 @@ func TestDomainStorageMapReadValue(t *testing.T) { }) t.Run("non-empty", func(t *testing.T) { + t.Parallel() + random := rand.New(rand.NewSource(42)) ledger := NewTestLedger(nil, nil) - storage := runtime.NewStorage(ledger, nil) + storage := runtime.NewStorage( + ledger, + nil, + runtime.StorageConfig{}, + ) // Turn off AtreeStorageValidationEnabled and explicitly check atree storage health at the end of test. // This is because DomainStorageMap isn't created through runtime.Storage, so there isn't any @@ -169,10 +193,16 @@ func TestDomainStorageMapSetAndUpdateValue(t *testing.T) { address := common.MustBytesToAddress([]byte{0x1}) t.Run("empty", func(t *testing.T) { + t.Parallel() + random := rand.New(rand.NewSource(42)) ledger := NewTestLedger(nil, nil) - storage := runtime.NewStorage(ledger, nil) + storage := runtime.NewStorage( + ledger, + nil, + runtime.StorageConfig{}, + ) // Turn off AtreeStorageValidationEnabled and explicitly check atree storage health at the end of test. // This is because AccountStorageMap isn't created through runtime.Storage, so there isn't any @@ -200,10 +230,16 @@ func TestDomainStorageMapSetAndUpdateValue(t *testing.T) { }) t.Run("non-empty", func(t *testing.T) { + t.Parallel() + random := rand.New(rand.NewSource(42)) ledger := NewTestLedger(nil, nil) - storage := runtime.NewStorage(ledger, nil) + storage := runtime.NewStorage( + ledger, + nil, + runtime.StorageConfig{}, + ) // Turn off AtreeStorageValidationEnabled and explicitly check atree storage health at the end of test. // This is because AccountStorageMap isn't created through runtime.Storage, so there isn't any @@ -245,8 +281,14 @@ func TestDomainStorageMapRemoveValue(t *testing.T) { address := common.MustBytesToAddress([]byte{0x1}) t.Run("empty", func(t *testing.T) { + t.Parallel() + ledger := NewTestLedger(nil, nil) - storage := runtime.NewStorage(ledger, nil) + storage := runtime.NewStorage( + ledger, + nil, + runtime.StorageConfig{}, + ) // Turn off AtreeStorageValidationEnabled and explicitly check atree storage health at the end of test. // This is because AccountStorageMap isn't created through runtime.Storage, so there isn't any @@ -273,10 +315,16 @@ func TestDomainStorageMapRemoveValue(t *testing.T) { }) t.Run("non-empty", func(t *testing.T) { + t.Parallel() + random := rand.New(rand.NewSource(42)) ledger := NewTestLedger(nil, nil) - storage := runtime.NewStorage(ledger, nil) + storage := runtime.NewStorage( + ledger, + nil, + runtime.StorageConfig{}, + ) // Turn off AtreeStorageValidationEnabled and explicitly check atree storage health at the end of test. // This is because AccountStorageMap isn't created through runtime.Storage, so there isn't any @@ -325,8 +373,14 @@ func TestDomainStorageMapIteratorNext(t *testing.T) { address := common.MustBytesToAddress([]byte{0x1}) t.Run("empty", func(t *testing.T) { + t.Parallel() + ledger := NewTestLedger(nil, nil) - storage := runtime.NewStorage(ledger, nil) + storage := runtime.NewStorage( + ledger, + nil, + runtime.StorageConfig{}, + ) // Turn off AtreeStorageValidationEnabled and explicitly check atree storage health at the end of test. // This is because AccountStorageMap isn't created through runtime.Storage, so there isn't any @@ -362,10 +416,16 @@ func TestDomainStorageMapIteratorNext(t *testing.T) { }) t.Run("non-empty", func(t *testing.T) { + t.Parallel() + random := rand.New(rand.NewSource(42)) ledger := NewTestLedger(nil, nil) - storage := runtime.NewStorage(ledger, nil) + storage := runtime.NewStorage( + ledger, + nil, + runtime.StorageConfig{}, + ) // Turn off AtreeStorageValidationEnabled and explicitly check atree storage health at the end of test. // This is because AccountStorageMap isn't created through runtime.Storage, so there isn't any @@ -422,8 +482,14 @@ func TestDomainStorageMapIteratorNextKey(t *testing.T) { address := common.MustBytesToAddress([]byte{0x1}) t.Run("empty", func(t *testing.T) { + t.Parallel() + ledger := NewTestLedger(nil, nil) - storage := runtime.NewStorage(ledger, nil) + storage := runtime.NewStorage( + ledger, + nil, + runtime.StorageConfig{}, + ) // Turn off AtreeStorageValidationEnabled and explicitly check atree storage health at the end of test. // This is because AccountStorageMap isn't created through runtime.Storage, so there isn't any @@ -458,10 +524,16 @@ func TestDomainStorageMapIteratorNextKey(t *testing.T) { }) t.Run("non-empty", func(t *testing.T) { + t.Parallel() + random := rand.New(rand.NewSource(42)) ledger := NewTestLedger(nil, nil) - storage := runtime.NewStorage(ledger, nil) + storage := runtime.NewStorage( + ledger, + nil, + runtime.StorageConfig{}, + ) // Turn off AtreeStorageValidationEnabled and explicitly check atree storage health at the end of test. // This is because AccountStorageMap isn't created through runtime.Storage, so there isn't any @@ -515,8 +587,14 @@ func TestDomainStorageMapIteratorNextValue(t *testing.T) { address := common.MustBytesToAddress([]byte{0x1}) t.Run("empty", func(t *testing.T) { + t.Parallel() + ledger := NewTestLedger(nil, nil) - storage := runtime.NewStorage(ledger, nil) + storage := runtime.NewStorage( + ledger, + nil, + runtime.StorageConfig{}, + ) // Turn off AtreeStorageValidationEnabled and explicitly check atree storage health at the end of test. // This is because AccountStorageMap isn't created through runtime.Storage, so there isn't any @@ -551,10 +629,16 @@ func TestDomainStorageMapIteratorNextValue(t *testing.T) { }) t.Run("non-empty", func(t *testing.T) { + t.Parallel() + random := rand.New(rand.NewSource(42)) ledger := NewTestLedger(nil, nil) - storage := runtime.NewStorage(ledger, nil) + storage := runtime.NewStorage( + ledger, + nil, + runtime.StorageConfig{}, + ) // Turn off AtreeStorageValidationEnabled and explicitly check atree storage health at the end of test. // This is because AccountStorageMap isn't created through runtime.Storage, so there isn't any @@ -615,9 +699,15 @@ func TestDomainStorageMapLoadFromRootSlabID(t *testing.T) { address := common.MustBytesToAddress([]byte{0x1}) t.Run("empty", func(t *testing.T) { + t.Parallel() + init := func() (atree.SlabID, domainStorageMapValues, map[string][]byte, map[string]uint64) { ledger := NewTestLedger(nil, nil) - storage := runtime.NewStorage(ledger, nil) + storage := runtime.NewStorage( + ledger, + nil, + runtime.StorageConfig{}, + ) inter := NewTestInterpreterWithStorage(t, storage) @@ -635,7 +725,11 @@ func TestDomainStorageMapLoadFromRootSlabID(t *testing.T) { domainStorageMapRootSlabID, domainValues, storedValues, storageIndices := init() ledger := NewTestLedgerWithData(nil, nil, storedValues, storageIndices) - storage := runtime.NewStorage(ledger, nil) + storage := runtime.NewStorage( + ledger, + nil, + runtime.StorageConfig{}, + ) domainStorageMap := interpreter.NewDomainStorageMapWithRootID(storage, domainStorageMapRootSlabID) require.Equal(t, uint64(0), domainStorageMap.Count()) @@ -648,12 +742,17 @@ func TestDomainStorageMapLoadFromRootSlabID(t *testing.T) { }) t.Run("non-empty", func(t *testing.T) { + t.Parallel() init := func() (atree.SlabID, domainStorageMapValues, map[string][]byte, map[string]uint64) { random := rand.New(rand.NewSource(42)) ledger := NewTestLedger(nil, nil) - storage := runtime.NewStorage(ledger, nil) + storage := runtime.NewStorage( + ledger, + nil, + runtime.StorageConfig{}, + ) // Turn off automatic AtreeStorageValidationEnabled and explicitly check atree storage health directly. // This is because AccountStorageMap isn't created through storage, so there isn't any account register to match AccountStorageMap root slab. @@ -674,7 +773,11 @@ func TestDomainStorageMapLoadFromRootSlabID(t *testing.T) { domainStorageMapRootSlabID, domainValues, storedValues, storageIndices := init() ledger := NewTestLedgerWithData(nil, nil, storedValues, storageIndices) - storage := runtime.NewStorage(ledger, nil) + storage := runtime.NewStorage( + ledger, + nil, + runtime.StorageConfig{}, + ) domainStorageMap := interpreter.NewDomainStorageMapWithRootID(storage, domainStorageMapRootSlabID) diff --git a/interpreter/interpreter.go b/interpreter/interpreter.go index 7b8abb0d78..4f0874e923 100644 --- a/interpreter/interpreter.go +++ b/interpreter/interpreter.go @@ -237,7 +237,12 @@ func (c TypeCodes) Merge(codes TypeCodes) { type Storage interface { atree.SlabStorage - GetStorageMap(inter *Interpreter, address common.Address, domain common.StorageDomain, createIfNotExists bool) *DomainStorageMap + GetDomainStorageMap( + inter *Interpreter, + address common.Address, + domain common.StorageDomain, + createIfNotExists bool, + ) *DomainStorageMap CheckHealth() error } @@ -2681,7 +2686,7 @@ func (interpreter *Interpreter) StoredValueExists( domain common.StorageDomain, identifier StorageMapKey, ) bool { - accountStorage := interpreter.Storage().GetStorageMap(interpreter, storageAddress, domain, false) + accountStorage := interpreter.Storage().GetDomainStorageMap(interpreter, storageAddress, domain, false) if accountStorage == nil { return false } @@ -2693,7 +2698,7 @@ func (interpreter *Interpreter) ReadStored( domain common.StorageDomain, identifier StorageMapKey, ) Value { - accountStorage := interpreter.Storage().GetStorageMap(interpreter, storageAddress, domain, false) + accountStorage := interpreter.Storage().GetDomainStorageMap(interpreter, storageAddress, domain, false) if accountStorage == nil { return nil } @@ -2706,7 +2711,7 @@ func (interpreter *Interpreter) WriteStored( key StorageMapKey, value Value, ) (existed bool) { - accountStorage := interpreter.Storage().GetStorageMap(interpreter, storageAddress, domain, true) + accountStorage := interpreter.Storage().GetDomainStorageMap(interpreter, storageAddress, domain, true) return accountStorage.WriteValue(interpreter, key, value) } @@ -4069,7 +4074,7 @@ func (interpreter *Interpreter) IsSubTypeOfSemaType(staticSubType StaticType, su } func (interpreter *Interpreter) domainPaths(address common.Address, domain common.PathDomain) []Value { - storageMap := interpreter.Storage().GetStorageMap(interpreter, address, domain.StorageDomain(), false) + storageMap := interpreter.Storage().GetDomainStorageMap(interpreter, address, domain.StorageDomain(), false) if storageMap == nil { return []Value{} } @@ -4164,7 +4169,7 @@ func (interpreter *Interpreter) newStorageIterationFunction( parameterTypes := fnType.ParameterTypes() returnType := fnType.ReturnTypeAnnotation.Type - storageMap := config.Storage.GetStorageMap(interpreter, address, domain.StorageDomain(), false) + storageMap := config.Storage.GetDomainStorageMap(interpreter, address, domain.StorageDomain(), false) if storageMap == nil { // if nothing is stored, no iteration is required return Void diff --git a/interpreter/misc_test.go b/interpreter/misc_test.go index f81454cbfc..25d1839941 100644 --- a/interpreter/misc_test.go +++ b/interpreter/misc_test.go @@ -5350,7 +5350,7 @@ func TestInterpretReferenceFailableDowncasting(t *testing.T) { ) domain := storagePath.Domain.StorageDomain() - storageMap := storage.GetStorageMap(inter, storageAddress, domain, true) + storageMap := storage.GetDomainStorageMap(inter, storageAddress, domain, true) storageMapKey := interpreter.StringStorageMapKey(storagePath.Identifier) storageMap.WriteValue(inter, storageMapKey, r) diff --git a/interpreter/storage.go b/interpreter/storage.go index e89417af39..1927cdfa0d 100644 --- a/interpreter/storage.go +++ b/interpreter/storage.go @@ -20,6 +20,7 @@ package interpreter import ( "bytes" + "cmp" "io" "math" "strings" @@ -106,6 +107,19 @@ type StorageDomainKey struct { Address common.Address } +func (k StorageDomainKey) Compare(o StorageDomainKey) int { + switch bytes.Compare(k.Address[:], o.Address[:]) { + case -1: + return -1 + case 0: + return cmp.Compare(k.Domain, o.Domain) + case 1: + return 1 + default: + panic(errors.NewUnreachableError()) + } +} + func NewStorageDomainKey( memoryGauge common.MemoryGauge, address common.Address, @@ -147,8 +161,8 @@ func (k StorageKey) IsLess(o StorageKey) bool { // InMemoryStorage type InMemoryStorage struct { *atree.BasicSlabStorage - StorageMaps map[StorageDomainKey]*DomainStorageMap - memoryGauge common.MemoryGauge + DomainStorageMaps map[StorageDomainKey]*DomainStorageMap + memoryGauge common.MemoryGauge } var _ Storage = InMemoryStorage{} @@ -174,27 +188,27 @@ func NewInMemoryStorage(memoryGauge common.MemoryGauge) InMemoryStorage { ) return InMemoryStorage{ - BasicSlabStorage: slabStorage, - StorageMaps: make(map[StorageDomainKey]*DomainStorageMap), - memoryGauge: memoryGauge, + BasicSlabStorage: slabStorage, + DomainStorageMaps: make(map[StorageDomainKey]*DomainStorageMap), + memoryGauge: memoryGauge, } } -func (i InMemoryStorage) GetStorageMap( +func (i InMemoryStorage) GetDomainStorageMap( _ *Interpreter, address common.Address, domain common.StorageDomain, createIfNotExists bool, ) ( - storageMap *DomainStorageMap, + domainStorageMap *DomainStorageMap, ) { key := NewStorageDomainKey(i.memoryGauge, address, domain) - storageMap = i.StorageMaps[key] - if storageMap == nil && createIfNotExists { - storageMap = NewDomainStorageMap(i.memoryGauge, i, atree.Address(address)) - i.StorageMaps[key] = storageMap + domainStorageMap = i.DomainStorageMaps[key] + if domainStorageMap == nil && createIfNotExists { + domainStorageMap = NewDomainStorageMap(i.memoryGauge, i, atree.Address(address)) + i.DomainStorageMaps[key] = domainStorageMap } - return storageMap + return domainStorageMap } func (i InMemoryStorage) CheckHealth() error { diff --git a/interpreter/storage_test.go b/interpreter/storage_test.go index 147385e051..fe822fd05b 100644 --- a/interpreter/storage_test.go +++ b/interpreter/storage_test.go @@ -524,7 +524,7 @@ func TestStorageOverwriteAndRemove(t *testing.T) { const storageMapKey = StringStorageMapKey("test") - storageMap := storage.GetStorageMap(inter, address, common.StorageDomainPathStorage, true) + storageMap := storage.GetDomainStorageMap(inter, address, common.StorageDomainPathStorage, true) storageMap.WriteValue(inter, storageMapKey, array1) // Overwriting delete any existing child slabs diff --git a/interpreter/stringatreevalue_test.go b/interpreter/stringatreevalue_test.go index bbeccd9414..ffaedf6d44 100644 --- a/interpreter/stringatreevalue_test.go +++ b/interpreter/stringatreevalue_test.go @@ -45,7 +45,7 @@ func TestLargeStringAtreeValueInSeparateSlab(t *testing.T) { ) require.NoError(t, err) - storageMap := storage.GetStorageMap( + storageMap := storage.GetDomainStorageMap( inter, common.MustBytesToAddress([]byte{0x1}), common.PathDomainStorage.StorageDomain(), diff --git a/interpreter/value_test.go b/interpreter/value_test.go index 09f1e233d7..3eaace67f8 100644 --- a/interpreter/value_test.go +++ b/interpreter/value_test.go @@ -3806,7 +3806,7 @@ func TestValue_ConformsToStaticType(t *testing.T) { ) require.NoError(t, err) - storageMap := storage.GetStorageMap(inter, testAddress, common.StorageDomainPathStorage, true) + storageMap := storage.GetDomainStorageMap(inter, testAddress, common.StorageDomainPathStorage, true) storageMap.WriteValue(inter, StringStorageMapKey("test"), TrueValue) value := valueFactory(inter) diff --git a/runtime/account_storage_v1.go b/runtime/account_storage_v1.go new file mode 100644 index 0000000000..53eec2fef2 --- /dev/null +++ b/runtime/account_storage_v1.go @@ -0,0 +1,212 @@ +/* + * Cadence - The resource-oriented smart contract programming language + * + * Copyright Flow Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package runtime + +import ( + "sort" + + "github.com/onflow/atree" + + "github.com/onflow/cadence/common" + "github.com/onflow/cadence/errors" + "github.com/onflow/cadence/interpreter" +) + +type AccountStorageV1 struct { + ledger atree.Ledger + slabStorage atree.SlabStorage + memoryGauge common.MemoryGauge + + // newDomainStorageMapSlabIndices contains root slab indices of new domain storage maps. + // The indices are saved using Ledger.SetValue() during commit(). + // Key is StorageDomainKey{common.StorageDomain, Address} and value is 8-byte slab index. + newDomainStorageMapSlabIndices map[interpreter.StorageDomainKey]atree.SlabIndex +} + +func NewAccountStorageV1( + ledger atree.Ledger, + slabStorage atree.SlabStorage, + memoryGauge common.MemoryGauge, +) *AccountStorageV1 { + return &AccountStorageV1{ + ledger: ledger, + slabStorage: slabStorage, + memoryGauge: memoryGauge, + } +} + +func (s *AccountStorageV1) GetDomainStorageMap( + address common.Address, + domain common.StorageDomain, + createIfNotExists bool, +) ( + domainStorageMap *interpreter.DomainStorageMap, +) { + var err error + domainStorageMap, err = getDomainStorageMapFromV1DomainRegister( + s.ledger, + s.slabStorage, + address, + domain, + ) + if err != nil { + panic(err) + } + + if domainStorageMap == nil && createIfNotExists { + domainStorageMap = s.storeNewDomainStorageMap(address, domain) + } + + return domainStorageMap +} + +func (s *AccountStorageV1) storeNewDomainStorageMap( + address common.Address, + domain common.StorageDomain, +) *interpreter.DomainStorageMap { + + domainStorageMap := interpreter.NewDomainStorageMap( + s.memoryGauge, + s.slabStorage, + atree.Address(address), + ) + + slabIndex := domainStorageMap.SlabID().Index() + + storageKey := interpreter.NewStorageDomainKey(s.memoryGauge, address, domain) + + if s.newDomainStorageMapSlabIndices == nil { + s.newDomainStorageMapSlabIndices = map[interpreter.StorageDomainKey]atree.SlabIndex{} + } + s.newDomainStorageMapSlabIndices[storageKey] = slabIndex + + return domainStorageMap +} + +func (s *AccountStorageV1) commit() error { + + switch len(s.newDomainStorageMapSlabIndices) { + case 0: + // Nothing to commit. + return nil + + case 1: + // Optimize for the common case of a single domain storage map. + + var updated int + for storageDomainKey, slabIndex := range s.newDomainStorageMapSlabIndices { //nolint:maprange + if updated > 0 { + panic(errors.NewUnreachableError()) + } + + err := s.writeStorageDomainSlabIndex( + storageDomainKey, + slabIndex, + ) + if err != nil { + return err + } + + updated++ + } + + default: + // Sort the indices to ensure deterministic order + + type domainStorageMapSlabIndex struct { + StorageDomainKey interpreter.StorageDomainKey + SlabIndex atree.SlabIndex + } + + slabIndices := make([]domainStorageMapSlabIndex, 0, len(s.newDomainStorageMapSlabIndices)) + for storageDomainKey, slabIndex := range s.newDomainStorageMapSlabIndices { //nolint:maprange + slabIndices = append( + slabIndices, + domainStorageMapSlabIndex{ + StorageDomainKey: storageDomainKey, + SlabIndex: slabIndex, + }, + ) + } + sort.Slice( + slabIndices, + func(i, j int) bool { + slabIndex1 := slabIndices[i] + slabIndex2 := slabIndices[j] + domainKey1 := slabIndex1.StorageDomainKey + domainKey2 := slabIndex2.StorageDomainKey + return domainKey1.Compare(domainKey2) < 0 + }, + ) + + for _, slabIndex := range slabIndices { + err := s.writeStorageDomainSlabIndex( + slabIndex.StorageDomainKey, + slabIndex.SlabIndex, + ) + if err != nil { + return err + } + } + } + + s.newDomainStorageMapSlabIndices = nil + + return nil +} + +func (s *AccountStorageV1) writeStorageDomainSlabIndex( + storageDomainKey interpreter.StorageDomainKey, + slabIndex atree.SlabIndex, +) error { + return writeSlabIndexToRegister( + s.ledger, + storageDomainKey.Address, + []byte(storageDomainKey.Domain.Identifier()), + slabIndex, + ) +} + +// getDomainStorageMapFromV1DomainRegister returns domain storage map from legacy domain register. +func getDomainStorageMapFromV1DomainRegister( + ledger atree.Ledger, + storage atree.SlabStorage, + address common.Address, + domain common.StorageDomain, +) (*interpreter.DomainStorageMap, error) { + + domainStorageSlabIndex, domainRegisterExists, err := readSlabIndexFromRegister( + ledger, + address, + []byte(domain.Identifier()), + ) + if err != nil { + return nil, err + } + if !domainRegisterExists { + return nil, nil + } + + slabID := atree.NewSlabID( + atree.Address(address), + domainStorageSlabIndex, + ) + + return interpreter.NewDomainStorageMapWithRootID(storage, slabID), nil +} diff --git a/runtime/account_storage_v2.go b/runtime/account_storage_v2.go new file mode 100644 index 0000000000..71e19fdaed --- /dev/null +++ b/runtime/account_storage_v2.go @@ -0,0 +1,319 @@ +/* + * Cadence - The resource-oriented smart contract programming language + * + * Copyright Flow Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package runtime + +import ( + "sort" + + "github.com/onflow/atree" + + "github.com/onflow/cadence/common" + "github.com/onflow/cadence/errors" + "github.com/onflow/cadence/interpreter" +) + +type AccountStorageV2 struct { + ledger atree.Ledger + slabStorage atree.SlabStorage + memoryGauge common.MemoryGauge + + // cachedAccountStorageMaps is a cache of account storage maps. + cachedAccountStorageMaps map[common.Address]*interpreter.AccountStorageMap + + // newAccountStorageMapSlabIndices contains root slab indices of new account storage maps. + // The indices are saved using Ledger.SetValue() during commit(). + newAccountStorageMapSlabIndices map[common.Address]atree.SlabIndex +} + +func NewAccountStorageV2( + ledger atree.Ledger, + slabStorage atree.SlabStorage, + memoryGauge common.MemoryGauge, +) *AccountStorageV2 { + return &AccountStorageV2{ + ledger: ledger, + slabStorage: slabStorage, + memoryGauge: memoryGauge, + } +} + +func (s *AccountStorageV2) GetDomainStorageMap( + inter *interpreter.Interpreter, + address common.Address, + domain common.StorageDomain, + createIfNotExists bool, +) ( + domainStorageMap *interpreter.DomainStorageMap, +) { + accountStorageMap := s.getAccountStorageMap(address) + + if accountStorageMap == nil && createIfNotExists { + accountStorageMap = s.storeNewAccountStorageMap(address) + } + + if accountStorageMap != nil { + domainStorageMap = accountStorageMap.GetDomain( + s.memoryGauge, + inter, + domain, + createIfNotExists, + ) + } + + return +} + +// getAccountStorageMap returns AccountStorageMap if exists, or nil otherwise. +func (s *AccountStorageV2) getAccountStorageMap( + address common.Address, +) ( + accountStorageMap *interpreter.AccountStorageMap, +) { + // Return cached account storage map if it exists. + + if s.cachedAccountStorageMaps != nil { + accountStorageMap = s.cachedAccountStorageMaps[address] + if accountStorageMap != nil { + return accountStorageMap + } + } + + defer func() { + if accountStorageMap != nil { + s.cacheAccountStorageMap( + address, + accountStorageMap, + ) + } + }() + + // Load account storage map if account storage register exists. + + var err error + accountStorageMap, err = getAccountStorageMapFromRegister( + s.ledger, + s.slabStorage, + address, + ) + if err != nil { + panic(err) + } + + return +} + +func (s *AccountStorageV2) cacheAccountStorageMap( + address common.Address, + accountStorageMap *interpreter.AccountStorageMap, +) { + if s.cachedAccountStorageMaps == nil { + s.cachedAccountStorageMaps = map[common.Address]*interpreter.AccountStorageMap{} + } + s.cachedAccountStorageMaps[address] = accountStorageMap +} + +func (s *AccountStorageV2) storeNewAccountStorageMap( + address common.Address, +) *interpreter.AccountStorageMap { + + accountStorageMap := interpreter.NewAccountStorageMap( + s.memoryGauge, + s.slabStorage, + atree.Address(address), + ) + + slabIndex := accountStorageMap.SlabID().Index() + + s.SetNewAccountStorageMapSlabIndex( + address, + slabIndex, + ) + + s.cacheAccountStorageMap( + address, + accountStorageMap, + ) + + return accountStorageMap +} + +func (s *AccountStorageV2) SetNewAccountStorageMapSlabIndex( + address common.Address, + slabIndex atree.SlabIndex, +) { + if s.newAccountStorageMapSlabIndices == nil { + s.newAccountStorageMapSlabIndices = map[common.Address]atree.SlabIndex{} + } + s.newAccountStorageMapSlabIndices[address] = slabIndex +} + +func (s *AccountStorageV2) commit() error { + switch len(s.newAccountStorageMapSlabIndices) { + case 0: + // Nothing to commit. + return nil + + case 1: + // Optimize for the common case of a single account storage map. + + var updated int + for address, slabIndex := range s.newAccountStorageMapSlabIndices { //nolint:maprange + if updated > 0 { + panic(errors.NewUnreachableError()) + } + + err := s.writeAccountStorageSlabIndex( + address, + slabIndex, + ) + if err != nil { + return err + } + + updated++ + } + + default: + // Sort the indices to ensure deterministic order + + type accountStorageMapSlabIndex struct { + Address common.Address + SlabIndex atree.SlabIndex + } + + slabIndices := make([]accountStorageMapSlabIndex, 0, len(s.newAccountStorageMapSlabIndices)) + for address, slabIndex := range s.newAccountStorageMapSlabIndices { //nolint:maprange + slabIndices = append( + slabIndices, + accountStorageMapSlabIndex{ + Address: address, + SlabIndex: slabIndex, + }, + ) + } + sort.Slice( + slabIndices, + func(i, j int) bool { + slabIndex1 := slabIndices[i] + slabIndex2 := slabIndices[j] + address1 := slabIndex1.Address + address2 := slabIndex2.Address + return address1.Compare(address2) < 0 + }, + ) + + for _, slabIndex := range slabIndices { + err := s.writeAccountStorageSlabIndex( + slabIndex.Address, + slabIndex.SlabIndex, + ) + if err != nil { + return err + } + } + } + + s.newAccountStorageMapSlabIndices = nil + + return nil +} + +func (s *AccountStorageV2) writeAccountStorageSlabIndex( + address common.Address, + slabIndex atree.SlabIndex, +) error { + return writeSlabIndexToRegister( + s.ledger, + address, + []byte(AccountStorageKey), + slabIndex, + ) +} + +func readAccountStorageSlabIndexFromRegister( + ledger atree.Ledger, + address common.Address, +) ( + atree.SlabIndex, + bool, + error, +) { + return readSlabIndexFromRegister( + ledger, + address, + []byte(AccountStorageKey), + ) +} + +func getAccountStorageMapFromRegister( + ledger atree.Ledger, + slabStorage atree.SlabStorage, + address common.Address, +) ( + *interpreter.AccountStorageMap, + error, +) { + slabIndex, registerExists, err := readAccountStorageSlabIndexFromRegister( + ledger, + address, + ) + if err != nil { + return nil, err + } + if !registerExists { + return nil, nil + } + + slabID := atree.NewSlabID( + atree.Address(address), + slabIndex, + ) + + return interpreter.NewAccountStorageMapWithRootID(slabStorage, slabID), nil +} + +func hasAccountStorageMap( + ledger atree.Ledger, + address common.Address, +) (bool, error) { + + _, registerExists, err := readAccountStorageSlabIndexFromRegister( + ledger, + address, + ) + if err != nil { + return false, err + } + return registerExists, nil +} + +func (s *AccountStorageV2) cachedRootSlabIDs() []atree.SlabID { + + var slabIDs []atree.SlabID + + // Get cached account storage map slab IDs. + for _, storageMap := range s.cachedAccountStorageMaps { //nolint:maprange + slabIDs = append( + slabIDs, + storageMap.SlabID(), + ) + } + + return slabIDs +} diff --git a/runtime/capabilitycontrollers_test.go b/runtime/capabilitycontrollers_test.go index feae867007..78ff497b96 100644 --- a/runtime/capabilitycontrollers_test.go +++ b/runtime/capabilitycontrollers_test.go @@ -3253,7 +3253,7 @@ func TestRuntimeCapabilityControllers(t *testing.T) { // Use *interpreter.Interpreter(nil) here because createIfNotExists is false. - storageMap := storage.GetStorageMap( + storageMap := storage.GetDomainStorageMap( nil, common.MustBytesToAddress([]byte{0x1}), common.StorageDomainPathCapability, @@ -3843,7 +3843,7 @@ func TestRuntimeCapabilitiesGetBackwardCompatibility(t *testing.T) { }) require.NoError(t, err) - publicStorageMap := storage.GetStorageMap( + publicStorageMap := storage.GetDomainStorageMap( inter, testAddress, common.PathDomainPublic.StorageDomain(), @@ -3951,7 +3951,7 @@ func TestRuntimeCapabilitiesPublishBackwardCompatibility(t *testing.T) { }) require.NoError(t, err) - publicStorageMap := storage.GetStorageMap( + publicStorageMap := storage.GetDomainStorageMap( inter, testAddress, common.PathDomainStorage.StorageDomain(), @@ -4042,7 +4042,7 @@ func TestRuntimeCapabilitiesUnpublishBackwardCompatibility(t *testing.T) { }) require.NoError(t, err) - publicStorageMap := storage.GetStorageMap( + publicStorageMap := storage.GetDomainStorageMap( inter, testAddress, common.PathDomainPublic.StorageDomain(), diff --git a/runtime/config.go b/runtime/config.go index d6882cb353..68926367d0 100644 --- a/runtime/config.go +++ b/runtime/config.go @@ -37,4 +37,6 @@ type Config struct { CoverageReport *CoverageReport // LegacyContractUpgradeEnabled enabled specifies whether to use the old parser when parsing an old contract LegacyContractUpgradeEnabled bool + // StorageFormatV2Enabled specifies whether storage format V2 is enabled + StorageFormatV2Enabled bool } diff --git a/runtime/contract_function_executor.go b/runtime/contract_function_executor.go index 8ba0f49bf8..19960185f2 100644 --- a/runtime/contract_function_executor.go +++ b/runtime/contract_function_executor.go @@ -105,7 +105,13 @@ func (executor *interpreterContractFunctionExecutor) preprocess() (err error) { runtimeInterface := context.Interface - storage := NewStorage(runtimeInterface, runtimeInterface) + storage := NewStorage( + runtimeInterface, + runtimeInterface, + StorageConfig{ + StorageFormatV2Enabled: interpreterRuntime.defaultConfig.StorageFormatV2Enabled, + }, + ) executor.storage = storage environment := context.Environment diff --git a/runtime/contract_test.go b/runtime/contract_test.go index 05a91e1b85..d62436af55 100644 --- a/runtime/contract_test.go +++ b/runtime/contract_test.go @@ -44,18 +44,21 @@ func TestRuntimeContract(t *testing.T) { t.Parallel() type testCase struct { - name string // the name of the contract used in add/update calls - code string // the code we use to add the contract - code2 string // the code we use to update the contract - valid bool - isInterface bool + name string // the name of the contract used in add/update calls + code string // the code we use to add the contract + code2 string // the code we use to update the contract + valid bool + isInterface bool + storageFormatV2Enabled bool } - test := func(t *testing.T, tc testCase) { - + runTest := func(t *testing.T, tc testCase) { t.Parallel() - runtime := NewTestInterpreterRuntime() + config := DefaultTestInterpreterConfig + config.StorageFormatV2Enabled = tc.storageFormatV2Enabled + + runtime := NewTestInterpreterRuntimeWithConfig(config) var loggedMessages []string @@ -222,8 +225,18 @@ func TestRuntimeContract(t *testing.T) { // so getting the storage map here once upfront would result in outdated data getContractValueExists := func() bool { - storageMap := NewStorage(storage, nil). - GetStorageMap(inter, signerAddress, common.StorageDomainContract, false) + storageMap := NewStorage( + storage, + nil, + StorageConfig{ + StorageFormatV2Enabled: tc.storageFormatV2Enabled, + }, + ).GetDomainStorageMap( + inter, + signerAddress, + common.StorageDomainContract, + false, + ) if storageMap == nil { return false } @@ -514,6 +527,18 @@ func TestRuntimeContract(t *testing.T) { } + test := func(t *testing.T, tc testCase) { + t.Run("storage format V2 disabled", func(t *testing.T) { + tc.storageFormatV2Enabled = false + runTest(t, tc) + }) + + t.Run("storage format V2 enabled", func(t *testing.T) { + tc.storageFormatV2Enabled = true + runTest(t, tc) + }) + } + t.Run("valid contract, correct name", func(t *testing.T) { test(t, testCase{ name: "Test", diff --git a/runtime/environment.go b/runtime/environment.go index 1ab7ee9b6a..6caa05fc8b 100644 --- a/runtime/environment.go +++ b/runtime/environment.go @@ -1106,7 +1106,7 @@ func (e *interpreterEnvironment) loadContract( location := compositeType.Location if addressLocation, ok := location.(common.AddressLocation); ok { - storageMap := e.storage.GetStorageMap( + storageMap := e.storage.GetDomainStorageMap( inter, addressLocation.Address, common.StorageDomainContract, diff --git a/runtime/ft_test.go b/runtime/ft_test.go index 56846cf2f5..144f062e1c 100644 --- a/runtime/ft_test.go +++ b/runtime/ft_test.go @@ -1083,7 +1083,7 @@ func TestRuntimeBrokenFungibleTokenRecovery(t *testing.T) { contractsAddress, ) - contractStorage := storage.GetStorageMap( + contractStorage := storage.GetDomainStorageMap( inter, contractsAddress, common.StorageDomainContract, @@ -1119,7 +1119,7 @@ func TestRuntimeBrokenFungibleTokenRecovery(t *testing.T) { userAddress, ) - userStorage := storage.GetStorageMap( + userStorage := storage.GetDomainStorageMap( inter, userAddress, common.PathDomainStorage.StorageDomain(), diff --git a/runtime/migrate_domain_registers.go b/runtime/migrate_domain_registers.go index e51bf81887..08d6afa55d 100644 --- a/runtime/migrate_domain_registers.go +++ b/runtime/migrate_domain_registers.go @@ -22,20 +22,21 @@ import ( "github.com/onflow/atree" "github.com/onflow/cadence/common" - "github.com/onflow/cadence/common/orderedmap" "github.com/onflow/cadence/errors" "github.com/onflow/cadence/interpreter" ) -type AccountStorageMaps = *orderedmap.OrderedMap[common.Address, *interpreter.AccountStorageMap] - type GetDomainStorageMapFunc func( ledger atree.Ledger, storage atree.SlabStorage, address common.Address, domain common.StorageDomain, -) (*interpreter.DomainStorageMap, error) +) ( + *interpreter.DomainStorageMap, + error, +) +// DomainRegisterMigration migrates domain registers to account storage maps. type DomainRegisterMigration struct { ledger atree.Ledger storage atree.SlabStorage @@ -49,76 +50,37 @@ func NewDomainRegisterMigration( storage atree.SlabStorage, inter *interpreter.Interpreter, memoryGauge common.MemoryGauge, + getDomainStorageMap GetDomainStorageMapFunc, ) *DomainRegisterMigration { + if getDomainStorageMap == nil { + getDomainStorageMap = getDomainStorageMapFromV1DomainRegister + } return &DomainRegisterMigration{ ledger: ledger, storage: storage, inter: inter, memoryGauge: memoryGauge, - getDomainStorageMap: getDomainStorageMapFromLegacyDomainRegister, + getDomainStorageMap: getDomainStorageMap, } } -// SetGetDomainStorageMapFunc allows user to provide custom GetDomainStorageMap function. -func (m *DomainRegisterMigration) SetGetDomainStorageMapFunc( - getDomainStorageMapFunc GetDomainStorageMapFunc, -) { - m.getDomainStorageMap = getDomainStorageMapFunc -} - -// MigrateAccounts migrates given accounts. -func (m *DomainRegisterMigration) MigrateAccounts( - accounts *orderedmap.OrderedMap[common.Address, struct{}], - pred func(common.Address) bool, +func (m *DomainRegisterMigration) MigrateAccount( + address common.Address, ) ( - AccountStorageMaps, + *interpreter.AccountStorageMap, error, ) { - if accounts == nil || accounts.Len() == 0 { - return nil, nil + exists, err := hasAccountStorageMap(m.ledger, address) + if err != nil { + return nil, err } - - var migratedAccounts AccountStorageMaps - - for pair := accounts.Oldest(); pair != nil; pair = pair.Next() { - address := pair.Key - - if !pred(address) { - continue - } - - migrated, err := isMigrated(m.ledger, address) - if err != nil { - return nil, err - } - if migrated { - continue - } - - accountStorageMap, err := m.MigrateAccount(address) - if err != nil { - return nil, err - } - - if accountStorageMap == nil { - continue - } - - if migratedAccounts == nil { - migratedAccounts = &orderedmap.OrderedMap[common.Address, *interpreter.AccountStorageMap]{} - } - migratedAccounts.Set(address, accountStorageMap) + if exists { + // Account storage map already exists + return nil, nil } - return migratedAccounts, nil -} - -func (m *DomainRegisterMigration) MigrateAccount( - address common.Address, -) (*interpreter.AccountStorageMap, error) { - // Migrate existing domains - accountStorageMap, err := m.migrateDomains(address) + accountStorageMap, err := m.migrateDomainRegisters(address) if err != nil { return nil, err } @@ -128,14 +90,14 @@ func (m *DomainRegisterMigration) MigrateAccount( return nil, nil } - accountStorageMapSlabIndex := accountStorageMap.SlabID().Index() + slabIndex := accountStorageMap.SlabID().Index() // Write account register errors.WrapPanic(func() { err = m.ledger.SetValue( address[:], []byte(AccountStorageKey), - accountStorageMapSlabIndex[:], + slabIndex[:], ) }) if err != nil { @@ -145,16 +107,25 @@ func (m *DomainRegisterMigration) MigrateAccount( return accountStorageMap, nil } -// migrateDomains migrates existing domain storage maps and removes domain registers. -func (m *DomainRegisterMigration) migrateDomains( +// migrateDomainRegisters migrates all existing domain storage maps to a new account storage map, +// and removes the domain registers. +func (m *DomainRegisterMigration) migrateDomainRegisters( address common.Address, -) (*interpreter.AccountStorageMap, error) { +) ( + *interpreter.AccountStorageMap, + error, +) { var accountStorageMap *interpreter.AccountStorageMap for _, domain := range common.AllStorageDomains { - domainStorageMap, err := m.getDomainStorageMap(m.ledger, m.storage, address, domain) + domainStorageMap, err := m.getDomainStorageMap( + m.ledger, + m.storage, + address, + domain, + ) if err != nil { return nil, err } @@ -165,7 +136,11 @@ func (m *DomainRegisterMigration) migrateDomains( } if accountStorageMap == nil { - accountStorageMap = interpreter.NewAccountStorageMap(m.memoryGauge, m.storage, atree.Address(address)) + accountStorageMap = interpreter.NewAccountStorageMap( + m.memoryGauge, + m.storage, + atree.Address(address), + ) } // Migrate (insert) existing domain storage map to account storage map @@ -194,11 +169,3 @@ func (m *DomainRegisterMigration) migrateDomains( return accountStorageMap, nil } - -func isMigrated(ledger atree.Ledger, address common.Address) (bool, error) { - _, registerExists, err := getSlabIndexFromRegisterValue(ledger, address, []byte(AccountStorageKey)) - if err != nil { - return false, err - } - return registerExists, nil -} diff --git a/runtime/migrate_domain_registers_test.go b/runtime/migrate_domain_registers_test.go index 2c18879dbe..f1d6e4304c 100644 --- a/runtime/migrate_domain_registers_test.go +++ b/runtime/migrate_domain_registers_test.go @@ -31,7 +31,6 @@ import ( "github.com/onflow/atree" "github.com/onflow/cadence/common" - "github.com/onflow/cadence/common/orderedmap" "github.com/onflow/cadence/errors" "github.com/onflow/cadence/interpreter" "github.com/onflow/cadence/runtime" @@ -40,20 +39,7 @@ import ( ) func TestMigrateDomainRegisters(t *testing.T) { - - alwaysMigrate := func(common.Address) bool { - return true - } - - neverMigrate := func(common.Address) bool { - return false - } - - migrateSpecificAccount := func(addressToMigrate common.Address) func(common.Address) bool { - return func(address common.Address) bool { - return address == addressToMigrate - } - } + t.Parallel() isAtreeRegister := func(key string) bool { return key[0] == '$' && len(key) == 9 @@ -73,47 +59,44 @@ func TestMigrateDomainRegisters(t *testing.T) { address1 := common.MustBytesToAddress([]byte{0x1}) address2 := common.MustBytesToAddress([]byte{0x2}) - t.Run("no accounts", func(t *testing.T) { - ledger := NewTestLedger(nil, nil) - storage := runtime.NewStorage(ledger, nil) - - inter := NewTestInterpreterWithStorage(t, storage) - - migrator := runtime.NewDomainRegisterMigration(ledger, storage, inter, nil) - - migratedAccounts, err := migrator.MigrateAccounts(nil, alwaysMigrate) - require.NoError(t, err) - require.True(t, migratedAccounts == nil || migratedAccounts.Len() == 0) - - err = storage.FastCommit(goruntime.NumCPU()) - require.NoError(t, err) - - require.Equal(t, 0, len(ledger.StoredValues)) - }) + addresses := []common.Address{address2, address1} t.Run("accounts without domain registers", func(t *testing.T) { + t.Parallel() + ledger := NewTestLedger(nil, nil) - storage := runtime.NewStorage(ledger, nil) + storage := runtime.NewStorage( + ledger, + nil, + runtime.StorageConfig{ + StorageFormatV2Enabled: true, + }, + ) inter := NewTestInterpreterWithStorage(t, storage) - migrator := runtime.NewDomainRegisterMigration(ledger, storage, inter, nil) - - accounts := &orderedmap.OrderedMap[common.Address, struct{}]{} - accounts.Set(address2, struct{}{}) - accounts.Set(address1, struct{}{}) + migrator := runtime.NewDomainRegisterMigration( + ledger, + storage, + inter, + nil, + nil, + ) - migratedAccounts, err := migrator.MigrateAccounts(accounts, alwaysMigrate) - require.NoError(t, err) - require.True(t, migratedAccounts == nil || migratedAccounts.Len() == 0) + for _, address := range addresses { + accountStorageMap, err := migrator.MigrateAccount(address) + require.Nil(t, accountStorageMap) + require.NoError(t, err) + } - err = storage.FastCommit(goruntime.NumCPU()) + err := storage.FastCommit(goruntime.NumCPU()) require.NoError(t, err) require.Equal(t, 0, len(ledger.StoredValues)) }) t.Run("accounts with domain registers", func(t *testing.T) { + t.Parallel() accountsInfo := []accountInfo{ { @@ -132,36 +115,44 @@ func TestMigrateDomainRegisters(t *testing.T) { } ledger, accountsValues := newTestLedgerWithUnmigratedAccounts(t, nil, nil, accountsInfo) - storage := runtime.NewStorage(ledger, nil) + storage := runtime.NewStorage( + ledger, + nil, + runtime.StorageConfig{ + StorageFormatV2Enabled: true, + }, + ) inter := NewTestInterpreterWithStorage(t, storage) - migrator := runtime.NewDomainRegisterMigration(ledger, storage, inter, nil) - - accounts := &orderedmap.OrderedMap[common.Address, struct{}]{} - accounts.Set(address2, struct{}{}) - accounts.Set(address1, struct{}{}) + migrator := runtime.NewDomainRegisterMigration( + ledger, + storage, + inter, + nil, + nil, + ) - migratedAccounts, err := migrator.MigrateAccounts(accounts, alwaysMigrate) - require.NoError(t, err) - require.NotNil(t, migratedAccounts) - require.Equal(t, accounts.Len(), migratedAccounts.Len()) - require.Equal(t, address2, migratedAccounts.Oldest().Key) - require.Equal(t, address1, migratedAccounts.Newest().Key) + var accountStorageMaps []*interpreter.AccountStorageMap + for _, address := range addresses { + accountStorageMap, err := migrator.MigrateAccount(address) + require.NotNil(t, accountStorageMap) + require.NoError(t, err) + accountStorageMaps = append(accountStorageMaps, accountStorageMap) + } - err = storage.FastCommit(goruntime.NumCPU()) + err := storage.FastCommit(goruntime.NumCPU()) require.NoError(t, err) // Check non-atree registers nonAtreeRegisters := getNonAtreeRegisters(ledger.StoredValues) - require.Equal(t, accounts.Len(), len(nonAtreeRegisters)) + require.Equal(t, len(addresses), len(nonAtreeRegisters)) require.Contains(t, nonAtreeRegisters, string(address1[:])+"|"+runtime.AccountStorageKey) require.Contains(t, nonAtreeRegisters, string(address2[:])+"|"+runtime.AccountStorageKey) // Check atree storage - expectedRootSlabIDs := make([]atree.SlabID, 0, migratedAccounts.Len()) - for pair := migratedAccounts.Oldest(); pair != nil; pair = pair.Next() { - accountStorageMap := pair.Value + expectedRootSlabIDs := make([]atree.SlabID, 0, len(accountStorageMaps)) + for _, accountStorageMap := range accountStorageMaps { expectedRootSlabIDs = append(expectedRootSlabIDs, accountStorageMap.SlabID()) } @@ -174,6 +165,8 @@ func TestMigrateDomainRegisters(t *testing.T) { }) t.Run("migrated accounts", func(t *testing.T) { + t.Parallel() + accountsInfo := []accountInfo{ { address: address1, @@ -191,93 +184,41 @@ func TestMigrateDomainRegisters(t *testing.T) { } ledger, accountsValues := newTestLedgerWithMigratedAccounts(t, nil, nil, accountsInfo) - storage := runtime.NewStorage(ledger, nil) + storage := runtime.NewStorage( + ledger, + nil, + runtime.StorageConfig{ + StorageFormatV2Enabled: true, + }, + ) inter := NewTestInterpreterWithStorage(t, storage) - migrator := runtime.NewDomainRegisterMigration(ledger, storage, inter, nil) - - accounts := &orderedmap.OrderedMap[common.Address, struct{}]{} - accounts.Set(address2, struct{}{}) - accounts.Set(address1, struct{}{}) + migrator := runtime.NewDomainRegisterMigration( + ledger, + storage, + inter, + nil, + nil, + ) - migratedAccounts, err := migrator.MigrateAccounts(accounts, alwaysMigrate) - require.NoError(t, err) - require.True(t, migratedAccounts == nil || migratedAccounts.Len() == 0) + for _, address := range addresses { + accountStorageMap, err := migrator.MigrateAccount(address) + require.Nil(t, accountStorageMap) + require.NoError(t, err) + } // Check account storage map data for address, accountValues := range accountsValues { - checkAccountStorageMapData(t, ledger.StoredValues, ledger.StorageIndices, address, accountValues) + checkAccountStorageMapData( + t, + ledger.StoredValues, + ledger.StorageIndices, + address, + accountValues, + ) } }) - - t.Run("never migration predicate", func(t *testing.T) { - - accountsInfo := []accountInfo{ - { - address: address1, - domains: []domainInfo{ - {domain: common.PathDomainStorage.StorageDomain(), domainStorageMapCount: 10, maxDepth: 3}, - }, - }, - { - address: address2, - domains: []domainInfo{ - {domain: common.PathDomainPublic.StorageDomain(), domainStorageMapCount: 10, maxDepth: 3}, - }, - }, - } - - ledger, _ := newTestLedgerWithUnmigratedAccounts(t, nil, nil, accountsInfo) - storage := runtime.NewStorage(ledger, nil) - - inter := NewTestInterpreterWithStorage(t, storage) - - migrator := runtime.NewDomainRegisterMigration(ledger, storage, inter, nil) - - accounts := &orderedmap.OrderedMap[common.Address, struct{}]{} - accounts.Set(address2, struct{}{}) - accounts.Set(address1, struct{}{}) - - migratedAccounts, err := migrator.MigrateAccounts(accounts, neverMigrate) - require.NoError(t, err) - require.True(t, migratedAccounts == nil || migratedAccounts.Len() == 0) - }) - - t.Run("selective migration predicate", func(t *testing.T) { - - accountsInfo := []accountInfo{ - { - address: address1, - domains: []domainInfo{ - {domain: common.PathDomainStorage.StorageDomain(), domainStorageMapCount: 10, maxDepth: 3}, - }, - }, - { - address: address2, - domains: []domainInfo{ - {domain: common.PathDomainPublic.StorageDomain(), domainStorageMapCount: 10, maxDepth: 3}, - }, - }, - } - - ledger, _ := newTestLedgerWithUnmigratedAccounts(t, nil, nil, accountsInfo) - storage := runtime.NewStorage(ledger, nil) - - inter := NewTestInterpreterWithStorage(t, storage) - - migrator := runtime.NewDomainRegisterMigration(ledger, storage, inter, nil) - - accounts := &orderedmap.OrderedMap[common.Address, struct{}]{} - accounts.Set(address2, struct{}{}) - accounts.Set(address1, struct{}{}) - - migratedAccounts, err := migrator.MigrateAccounts(accounts, migrateSpecificAccount(address2)) - require.NoError(t, err) - require.NotNil(t, migratedAccounts) - require.Equal(t, 1, migratedAccounts.Len()) - require.Equal(t, address2, migratedAccounts.Oldest().Key) - }) } type domainInfo struct { @@ -298,7 +239,13 @@ func newTestLedgerWithUnmigratedAccounts( accounts []accountInfo, ) (TestLedger, map[common.Address]accountStorageMapValues) { ledger := NewTestLedger(nil, nil) - storage := runtime.NewStorage(ledger, nil) + storage := runtime.NewStorage( + ledger, + nil, + runtime.StorageConfig{ + StorageFormatV2Enabled: true, + }, + ) // Turn off AtreeStorageValidationEnabled and explicitly check atree storage health at the end of test. // This is because DomainStorageMap isn't created through runtime.Storage, so there isn't any @@ -382,7 +329,13 @@ func newTestLedgerWithMigratedAccounts( accounts []accountInfo, ) (TestLedger, map[common.Address]accountStorageMapValues) { ledger := NewTestLedger(nil, nil) - storage := runtime.NewStorage(ledger, nil) + storage := runtime.NewStorage( + ledger, + nil, + runtime.StorageConfig{ + StorageFormatV2Enabled: true, + }, + ) // Turn off AtreeStorageValidationEnabled and explicitly check atree storage health at the end of test. // This is because DomainStorageMap isn't created through runtime.Storage, so there isn't any diff --git a/runtime/runtime.go b/runtime/runtime.go index c6277c55ae..d516127ac8 100644 --- a/runtime/runtime.go +++ b/runtime/runtime.go @@ -558,7 +558,15 @@ func (r *interpreterRuntime) Storage(context Context) (*Storage, *interpreter.In codesAndPrograms := NewCodesAndPrograms() - storage := NewStorage(context.Interface, context.Interface) + runtimeInterface := context.Interface + + storage := NewStorage( + runtimeInterface, + runtimeInterface, + StorageConfig{ + StorageFormatV2Enabled: r.defaultConfig.StorageFormatV2Enabled, + }, + ) environment := context.Environment if environment == nil { @@ -566,7 +574,7 @@ func (r *interpreterRuntime) Storage(context Context) (*Storage, *interpreter.In } environment.Configure( - context.Interface, + runtimeInterface, codesAndPrograms, storage, context.CoverageReport, diff --git a/runtime/runtime_memory_metering_test.go b/runtime/runtime_memory_metering_test.go index a141120e99..6e6687e4c8 100644 --- a/runtime/runtime_memory_metering_test.go +++ b/runtime/runtime_memory_metering_test.go @@ -815,7 +815,9 @@ func TestRuntimeStorageCommitsMetering(t *testing.T) { // Before the storageUsed function is invoked, the deltas must have been committed. // So the encoded slabs must have been metered at this point. assert.Equal(t, uint64(0), meter.getMemory(common.MemoryKindAtreeEncodedSlab)) + storageUsedInvoked = true + return 1, nil }, } @@ -840,85 +842,152 @@ func TestRuntimeStorageCommitsMetering(t *testing.T) { t.Run("account.storage.save", func(t *testing.T) { t.Parallel() - code := []byte(` - transaction { - prepare(signer: auth(Storage) &Account) { - signer.storage.save([[1, 2, 3], [4, 5, 6]], to: /storage/test) - } - } - `) + test := func(storageFormatV2Enabled bool) { - meter := newTestMemoryGauge() + name := fmt.Sprintf( + "storage format V2 enabled: %v", + storageFormatV2Enabled, + ) - runtimeInterface := &TestRuntimeInterface{ - Storage: NewTestLedger(nil, nil), - OnGetSigningAccounts: func() ([]Address, error) { - return []Address{{42}}, nil - }, - OnMeterMemory: meter.MeterMemory, - } + t.Run(name, func(t *testing.T) { + t.Parallel() - runtime := NewTestInterpreterRuntime() + code := []byte(` + transaction { + prepare(signer: auth(Storage) &Account) { + signer.storage.save([[1, 2, 3], [4, 5, 6]], to: /storage/test) + } + } + `) - err := runtime.ExecuteTransaction( - Script{ - Source: code, - }, - Context{ - Interface: runtimeInterface, - Location: common.TransactionLocation{}, - }, - ) + meter := newTestMemoryGauge() - require.NoError(t, err) - assert.Equal(t, uint64(5), meter.getMemory(common.MemoryKindAtreeEncodedSlab)) + runtimeInterface := &TestRuntimeInterface{ + Storage: NewTestLedger(nil, nil), + OnGetSigningAccounts: func() ([]Address, error) { + return []Address{{42}}, nil + }, + OnMeterMemory: meter.MeterMemory, + } + + config := DefaultTestInterpreterConfig + config.StorageFormatV2Enabled = storageFormatV2Enabled + runtime := NewTestInterpreterRuntimeWithConfig(config) + + err := runtime.ExecuteTransaction( + Script{ + Source: code, + }, + Context{ + Interface: runtimeInterface, + Location: common.TransactionLocation{}, + }, + ) + + require.NoError(t, err) + + var expected uint64 + if storageFormatV2Enabled { + expected = 5 + } else { + expected = 4 + } + assert.Equal(t, + expected, + meter.getMemory(common.MemoryKindAtreeEncodedSlab), + ) + }) + } + + for _, storageFormatV2Enabled := range []bool{false, true} { + test(storageFormatV2Enabled) + } }) t.Run("storage used non empty", func(t *testing.T) { t.Parallel() - code := []byte(` - transaction { - prepare(signer: auth(Storage) &Account) { - signer.storage.save([[1, 2, 3], [4, 5, 6]], to: /storage/test) - signer.storage.used - } - } - `) + test := func(storageFormatV2Enabled bool) { - meter := newTestMemoryGauge() - storageUsedInvoked := false + name := fmt.Sprintf( + "storage format V2 enabled: %v", + storageFormatV2Enabled, + ) - runtimeInterface := &TestRuntimeInterface{ - Storage: NewTestLedger(nil, nil), - OnGetSigningAccounts: func() ([]Address, error) { - return []Address{{42}}, nil - }, - OnMeterMemory: meter.MeterMemory, - OnGetStorageUsed: func(_ Address) (uint64, error) { - // Before the storageUsed function is invoked, the deltas must have been committed. - // So the encoded slabs must have been metered at this point. - assert.Equal(t, uint64(5), meter.getMemory(common.MemoryKindAtreeEncodedSlab)) - storageUsedInvoked = true - return 1, nil - }, - } + t.Run(name, func(t *testing.T) { + t.Parallel() - runtime := NewTestInterpreterRuntime() + code := []byte(` + transaction { + prepare(signer: auth(Storage) &Account) { + signer.storage.save([[1, 2, 3], [4, 5, 6]], to: /storage/test) + signer.storage.used + } + } + `) - err := runtime.ExecuteTransaction( - Script{ - Source: code, - }, - Context{ - Interface: runtimeInterface, - Location: common.TransactionLocation{}, - }, - ) + meter := newTestMemoryGauge() + storageUsedInvoked := false - require.NoError(t, err) - assert.True(t, storageUsedInvoked) - assert.Equal(t, uint64(5), meter.getMemory(common.MemoryKindAtreeEncodedSlab)) + runtimeInterface := &TestRuntimeInterface{ + Storage: NewTestLedger(nil, nil), + OnGetSigningAccounts: func() ([]Address, error) { + return []Address{{42}}, nil + }, + OnMeterMemory: meter.MeterMemory, + OnGetStorageUsed: func(_ Address) (uint64, error) { + // Before the storageUsed function is invoked, the deltas must have been committed. + // So the encoded slabs must have been metered at this point. + var expected uint64 + if storageFormatV2Enabled { + expected = 5 + } else { + expected = 4 + } + assert.Equal(t, + expected, + meter.getMemory(common.MemoryKindAtreeEncodedSlab), + ) + + storageUsedInvoked = true + + return 1, nil + }, + } + + config := DefaultTestInterpreterConfig + config.StorageFormatV2Enabled = storageFormatV2Enabled + runtime := NewTestInterpreterRuntimeWithConfig(config) + + err := runtime.ExecuteTransaction( + Script{ + Source: code, + }, + Context{ + Interface: runtimeInterface, + Location: common.TransactionLocation{}, + }, + ) + + require.NoError(t, err) + assert.True(t, storageUsedInvoked) + + var expected uint64 + if storageFormatV2Enabled { + expected = 5 + } else { + expected = 4 + } + assert.Equal(t, + expected, + meter.getMemory(common.MemoryKindAtreeEncodedSlab), + ) + }) + } + + for _, storageFormatV2Enabled := range []bool{false, true} { + test(storageFormatV2Enabled) + } }) } @@ -1036,143 +1105,226 @@ func TestRuntimeMeterEncoding(t *testing.T) { t.Parallel() - config := DefaultTestInterpreterConfig - config.AtreeValidationEnabled = false - rt := NewTestInterpreterRuntimeWithConfig(config) + test := func(storageFormatV2Enabled bool) { - address := common.MustBytesToAddress([]byte{0x1}) - storage := NewTestLedger(nil, nil) - meter := newTestMemoryGauge() + name := fmt.Sprintf( + "storage format V2 enabled: %v", + storageFormatV2Enabled, + ) - runtimeInterface := &TestRuntimeInterface{ - Storage: storage, - OnGetSigningAccounts: func() ([]Address, error) { - return []Address{address}, nil - }, - OnMeterMemory: meter.MeterMemory, - } + t.Run(name, func(t *testing.T) { + t.Parallel() - text := "A quick brown fox jumps over the lazy dog" + config := DefaultTestInterpreterConfig + config.AtreeValidationEnabled = false + config.StorageFormatV2Enabled = storageFormatV2Enabled + rt := NewTestInterpreterRuntimeWithConfig(config) - err := rt.ExecuteTransaction( - Script{ - Source: []byte(fmt.Sprintf(` - transaction() { - prepare(acc: auth(Storage) &Account) { - var s = "%s" - acc.storage.save(s, to:/storage/some_path) - } - }`, - text, - )), - }, - Context{ - Interface: runtimeInterface, - Location: common.TransactionLocation{}, - }, - ) + address := common.MustBytesToAddress([]byte{0x1}) + storage := NewTestLedger(nil, nil) + meter := newTestMemoryGauge() - require.NoError(t, err) - assert.Equal(t, 107, int(meter.getMemory(common.MemoryKindBytes))) + runtimeInterface := &TestRuntimeInterface{ + Storage: storage, + OnGetSigningAccounts: func() ([]Address, error) { + return []Address{address}, nil + }, + OnMeterMemory: meter.MeterMemory, + } + + text := "A quick brown fox jumps over the lazy dog" + + err := rt.ExecuteTransaction( + Script{ + Source: []byte(fmt.Sprintf(` + transaction() { + prepare(acc: auth(Storage) &Account) { + var s = "%s" + acc.storage.save(s, to:/storage/some_path) + } + }`, + text, + )), + }, + Context{ + Interface: runtimeInterface, + Location: common.TransactionLocation{}, + }, + ) + + require.NoError(t, err) + + var expected uint64 + if storageFormatV2Enabled { + expected = 107 + } else { + expected = 75 + } + assert.Equal(t, + expected, + meter.getMemory(common.MemoryKindBytes), + ) + }) + } + + for _, storageFormatV2Enabled := range []bool{false, true} { + test(storageFormatV2Enabled) + } }) t.Run("string in loop", func(t *testing.T) { t.Parallel() - config := DefaultTestInterpreterConfig - config.AtreeValidationEnabled = false - rt := NewTestInterpreterRuntimeWithConfig(config) + test := func(storageFormatV2Enabled bool) { - address := common.MustBytesToAddress([]byte{0x1}) - storage := NewTestLedger(nil, nil) - meter := newTestMemoryGauge() + name := fmt.Sprintf( + "storage format V2 enabled: %v", + storageFormatV2Enabled, + ) - runtimeInterface := &TestRuntimeInterface{ - Storage: storage, - OnGetSigningAccounts: func() ([]Address, error) { - return []Address{address}, nil - }, - OnMeterMemory: meter.MeterMemory, - } + t.Run(name, func(t *testing.T) { + t.Parallel() - text := "A quick brown fox jumps over the lazy dog" + config := DefaultTestInterpreterConfig + config.AtreeValidationEnabled = false + config.StorageFormatV2Enabled = storageFormatV2Enabled + rt := NewTestInterpreterRuntimeWithConfig(config) - err := rt.ExecuteTransaction( - Script{ - Source: []byte(fmt.Sprintf(` - transaction() { - prepare(acc: auth(Storage) &Account) { - var i = 0 - var s = "%s" - while i<1000 { - let path = StoragePath(identifier: "i".concat(i.toString()))! - acc.storage.save(s, to: path) - i=i+1 - } - } - }`, - text, - )), - }, - Context{ - Interface: runtimeInterface, - Location: common.TransactionLocation{}, - }, - ) + address := common.MustBytesToAddress([]byte{0x1}) + storage := NewTestLedger(nil, nil) + meter := newTestMemoryGauge() - require.NoError(t, err) - assert.Equal(t, 61494, int(meter.getMemory(common.MemoryKindBytes))) + runtimeInterface := &TestRuntimeInterface{ + Storage: storage, + OnGetSigningAccounts: func() ([]Address, error) { + return []Address{address}, nil + }, + OnMeterMemory: meter.MeterMemory, + } + + text := "A quick brown fox jumps over the lazy dog" + + err := rt.ExecuteTransaction( + Script{ + Source: []byte(fmt.Sprintf(` + transaction() { + prepare(acc: auth(Storage) &Account) { + var i = 0 + var s = "%s" + while i<1000 { + let path = StoragePath(identifier: "i".concat(i.toString()))! + acc.storage.save(s, to: path) + i=i+1 + } + } + }`, + text, + )), + }, + Context{ + Interface: runtimeInterface, + Location: common.TransactionLocation{}, + }, + ) + + require.NoError(t, err) + + var expected uint64 + if storageFormatV2Enabled { + expected = 61494 + } else { + expected = 61455 + } + assert.Equal(t, + expected, + meter.getMemory(common.MemoryKindBytes), + ) + }) + } + + for _, storageFormatV2Enabled := range []bool{false, true} { + test(storageFormatV2Enabled) + } }) t.Run("composite", func(t *testing.T) { t.Parallel() - config := DefaultTestInterpreterConfig - config.AtreeValidationEnabled = false - rt := NewTestInterpreterRuntimeWithConfig(config) + test := func(storageFormatV2Enabled bool) { - address := common.MustBytesToAddress([]byte{0x1}) - storage := NewTestLedger(nil, nil) - meter := newTestMemoryGauge() + name := fmt.Sprintf( + "storage format V2 enabled: %v", + storageFormatV2Enabled, + ) - runtimeInterface := &TestRuntimeInterface{ - Storage: storage, - OnGetSigningAccounts: func() ([]Address, error) { - return []Address{address}, nil - }, - OnMeterMemory: meter.MeterMemory, - } + t.Run(name, func(t *testing.T) { + t.Parallel() - _, err := rt.ExecuteScript( - Script{ - Source: []byte(` - access(all) fun main() { - let acc = getAuthAccount(0x02) - var i = 0 - var f = Foo() - while i<1000 { - let path = StoragePath(identifier: "i".concat(i.toString()))! - acc.storage.save(f, to: path) - i=i+1 - } - } + config := DefaultTestInterpreterConfig + config.AtreeValidationEnabled = false + config.StorageFormatV2Enabled = storageFormatV2Enabled + rt := NewTestInterpreterRuntimeWithConfig(config) - access(all) struct Foo { - access(self) var id: Int - init() { - self.id = 123456789 - } - }`), - }, - Context{ - Interface: runtimeInterface, - Location: common.ScriptLocation{}, - }, - ) + address := common.MustBytesToAddress([]byte{0x1}) + storage := NewTestLedger(nil, nil) + meter := newTestMemoryGauge() - require.NoError(t, err) - assert.Equal(t, 58362, int(meter.getMemory(common.MemoryKindBytes))) + runtimeInterface := &TestRuntimeInterface{ + Storage: storage, + OnGetSigningAccounts: func() ([]Address, error) { + return []Address{address}, nil + }, + OnMeterMemory: meter.MeterMemory, + } + + _, err := rt.ExecuteScript( + Script{ + Source: []byte(` + access(all) fun main() { + let acc = getAuthAccount(0x02) + var i = 0 + var f = Foo() + while i<1000 { + let path = StoragePath(identifier: "i".concat(i.toString()))! + acc.storage.save(f, to: path) + i=i+1 + } + } + + access(all) struct Foo { + access(self) var id: Int + init() { + self.id = 123456789 + } + } + `), + }, + Context{ + Interface: runtimeInterface, + Location: common.ScriptLocation{}, + }, + ) + + require.NoError(t, err) + + var expected uint64 + if storageFormatV2Enabled { + expected = 58362 + } else { + expected = 58323 + } + + assert.Equal(t, + expected, + meter.getMemory(common.MemoryKindBytes), + ) + }) + } + + for _, storageFormatV2Enabled := range []bool{false, true} { + test(storageFormatV2Enabled) + } }) } diff --git a/runtime/runtime_test.go b/runtime/runtime_test.go index 702def3041..fbcc4a0716 100644 --- a/runtime/runtime_test.go +++ b/runtime/runtime_test.go @@ -5683,102 +5683,190 @@ func TestRuntimeContractWriteback(t *testing.T) { t.Parallel() - runtime := NewTestInterpreterRuntime() - addressValue := cadence.BytesToAddress([]byte{0xCA, 0xDE}) - contract := []byte(` - access(all) contract Test { + test := func( + storageFormatV2Enabled bool, + expectedDeployTxWrites []ownerKeyPair, + expectedWriteTxWrites []ownerKeyPair, + ) { - access(all) var test: Int + name := fmt.Sprintf( + "storage format V2 enabled: %v", + storageFormatV2Enabled, + ) - init() { - self.test = 1 - } + t.Run(name, func(t *testing.T) { + t.Parallel() - access(all) fun setTest(_ test: Int) { - self.test = test - } - } - `) + config := DefaultTestInterpreterConfig + config.StorageFormatV2Enabled = storageFormatV2Enabled + runtime := NewTestInterpreterRuntimeWithConfig(config) - deploy := DeploymentTransaction("Test", contract) + contract := []byte(` + access(all) contract Test { - readTx := []byte(` - import Test from 0xCADE + access(all) var test: Int - transaction { + init() { + self.test = 1 + } - prepare(signer: &Account) { - log(Test.test) - } - } - `) + access(all) fun setTest(_ test: Int) { + self.test = test + } + } + `) - writeTx := []byte(` - import Test from 0xCADE + deploy := DeploymentTransaction("Test", contract) - transaction { + readTx := []byte(` + import Test from 0xCADE - prepare(signer: &Account) { - Test.setTest(2) - } - } - `) + transaction { - var accountCode []byte - var events []cadence.Event - var loggedMessages []string - var writes []ownerKeyPair + prepare(signer: &Account) { + log(Test.test) + } + } + `) + + writeTx := []byte(` + import Test from 0xCADE + + transaction { + + prepare(signer: &Account) { + Test.setTest(2) + } + } + `) + + var accountCode []byte + var events []cadence.Event + var loggedMessages []string + var writes []ownerKeyPair + + onWrite := func(owner, key, value []byte) { + writes = append(writes, ownerKeyPair{ + owner, + key, + }) + } + + runtimeInterface := &TestRuntimeInterface{ + OnGetCode: func(_ Location) (bytes []byte, err error) { + return accountCode, nil + }, + Storage: NewTestLedger(nil, onWrite), + OnGetSigningAccounts: func() ([]Address, error) { + return []Address{Address(addressValue)}, nil + }, + OnResolveLocation: NewSingleIdentifierLocationResolver(t), + OnGetAccountContractCode: func(_ common.AddressLocation) (code []byte, err error) { + return accountCode, nil + }, + OnUpdateAccountContractCode: func(_ common.AddressLocation, code []byte) (err error) { + accountCode = code + return nil + }, + OnEmitEvent: func(event cadence.Event) error { + events = append(events, event) + return nil + }, + OnProgramLog: func(message string) { + loggedMessages = append(loggedMessages, message) + }, + } + + nextTransactionLocation := NewTransactionLocationGenerator() + + err := runtime.ExecuteTransaction( + Script{ + Source: deploy, + }, + Context{ + Interface: runtimeInterface, + Location: nextTransactionLocation(), + }, + ) + require.NoError(t, err) + + assert.NotNil(t, accountCode) + + assert.Equal(t, + expectedDeployTxWrites, + writes, + ) + + writes = nil + + err = runtime.ExecuteTransaction( + Script{ + Source: readTx, + }, + Context{ + Interface: runtimeInterface, + Location: nextTransactionLocation(), + }, + ) + require.NoError(t, err) + + assert.Empty(t, writes) + + writes = nil + + err = runtime.ExecuteTransaction( + Script{ + Source: writeTx, + }, + Context{ + Interface: runtimeInterface, + Location: nextTransactionLocation(), + }, + ) + require.NoError(t, err) + + assert.Equal(t, + expectedWriteTxWrites, + writes, + ) - onWrite := func(owner, key, value []byte) { - writes = append(writes, ownerKeyPair{ - owner, - key, }) } - runtimeInterface := &TestRuntimeInterface{ - OnGetCode: func(_ Location) (bytes []byte, err error) { - return accountCode, nil - }, - Storage: NewTestLedger(nil, onWrite), - OnGetSigningAccounts: func() ([]Address, error) { - return []Address{Address(addressValue)}, nil - }, - OnResolveLocation: NewSingleIdentifierLocationResolver(t), - OnGetAccountContractCode: func(_ common.AddressLocation) (code []byte, err error) { - return accountCode, nil - }, - OnUpdateAccountContractCode: func(_ common.AddressLocation, code []byte) (err error) { - accountCode = code - return nil - }, - OnEmitEvent: func(event cadence.Event) error { - events = append(events, event) - return nil - }, - OnProgramLog: func(message string) { - loggedMessages = append(loggedMessages, message) + test(false, + []ownerKeyPair{ + // storage index to contract domain storage map + { + addressValue[:], + []byte("contract"), + }, + // contract value + { + addressValue[:], + []byte{'$', 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1}, + }, + // contract domain storage map + { + addressValue[:], + []byte{'$', 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2}, + }, }, - } - - nextTransactionLocation := NewTransactionLocationGenerator() - err := runtime.ExecuteTransaction( - Script{ - Source: deploy, - }, - Context{ - Interface: runtimeInterface, - Location: nextTransactionLocation(), + []ownerKeyPair{ + // Storage map is modified because contract value is inlined in contract storage map. + // NOTE: contract value slab doesn't exist. + { + addressValue[:], + []byte{'$', 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2}, + }, }, ) - require.NoError(t, err) - assert.NotNil(t, accountCode) + test( + true, - assert.Equal(t, []ownerKeyPair{ // storage index to account storage map { @@ -5803,38 +5891,7 @@ func TestRuntimeContractWriteback(t *testing.T) { []byte{'$', 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3}, }, }, - writes, - ) - - writes = nil - err = runtime.ExecuteTransaction( - Script{ - Source: readTx, - }, - Context{ - Interface: runtimeInterface, - Location: nextTransactionLocation(), - }, - ) - require.NoError(t, err) - - assert.Empty(t, writes) - - writes = nil - - err = runtime.ExecuteTransaction( - Script{ - Source: writeTx, - }, - Context{ - Interface: runtimeInterface, - Location: nextTransactionLocation(), - }, - ) - require.NoError(t, err) - - assert.Equal(t, []ownerKeyPair{ // Account storage map is modified because: // - contract value is inlined in contract storage map, and @@ -5845,7 +5902,6 @@ func TestRuntimeContractWriteback(t *testing.T) { []byte{'$', 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2}, }, }, - writes, ) } @@ -5853,88 +5909,244 @@ func TestRuntimeStorageWriteback(t *testing.T) { t.Parallel() - runtime := NewTestInterpreterRuntime() - addressValue := cadence.BytesToAddress([]byte{0xCA, 0xDE}) - contract := []byte(` - access(all) contract Test { + test := func( + storageFormatV2Enabled bool, + expectedDeployTxWrites []ownerKeyPair, + expectedSaveToStorageTxWrites []ownerKeyPair, + expectedModifyStorageTxWrites []ownerKeyPair, + ) { - access(all) resource R { + name := fmt.Sprintf( + "storage format V2 enabled: %v", + storageFormatV2Enabled, + ) + t.Run(name, func(t *testing.T) { + t.Parallel() - access(all) var test: Int + config := DefaultTestInterpreterConfig + config.StorageFormatV2Enabled = storageFormatV2Enabled + runtime := NewTestInterpreterRuntimeWithConfig(config) + + contract := []byte(` + access(all) contract Test { + + access(all) resource R { + + access(all) var test: Int + + init() { + self.test = 1 + } + + access(all) fun setTest(_ test: Int) { + self.test = test + } + } - init() { - self.test = 1 + + access(all) fun createR(): @R { + return <-create R() + } } + `) - access(all) fun setTest(_ test: Int) { - self.test = test - } - } + deploy := DeploymentTransaction("Test", contract) + var accountCode []byte + var events []cadence.Event + var loggedMessages []string + var writes []ownerKeyPair - access(all) fun createR(): @R { - return <-create R() - } - } - `) + onWrite := func(owner, key, _ []byte) { + writes = append(writes, ownerKeyPair{ + owner, + key, + }) + } - deploy := DeploymentTransaction("Test", contract) + runtimeInterface := &TestRuntimeInterface{ + OnGetCode: func(_ Location) (bytes []byte, err error) { + return accountCode, nil + }, + Storage: NewTestLedger(nil, onWrite), + OnGetSigningAccounts: func() ([]Address, error) { + return []Address{Address(addressValue)}, nil + }, + OnResolveLocation: NewSingleIdentifierLocationResolver(t), + OnGetAccountContractCode: func(location common.AddressLocation) (code []byte, err error) { + return accountCode, nil + }, + OnUpdateAccountContractCode: func(location common.AddressLocation, code []byte) error { + accountCode = code + return nil + }, + OnEmitEvent: func(event cadence.Event) error { + events = append(events, event) + return nil + }, + OnProgramLog: func(message string) { + loggedMessages = append(loggedMessages, message) + }, + } - var accountCode []byte - var events []cadence.Event - var loggedMessages []string - var writes []ownerKeyPair + nextTransactionLocation := NewTransactionLocationGenerator() - onWrite := func(owner, key, _ []byte) { - writes = append(writes, ownerKeyPair{ - owner, - key, + err := runtime.ExecuteTransaction( + Script{ + Source: deploy, + }, + Context{ + Interface: runtimeInterface, + Location: nextTransactionLocation(), + }, + ) + require.NoError(t, err) + + assert.NotNil(t, accountCode) + + assert.Equal(t, + expectedDeployTxWrites, + writes, + ) + + writes = nil + + err = runtime.ExecuteTransaction( + Script{ + Source: []byte(` + import Test from 0xCADE + + transaction { + + prepare(signer: auth(Storage) &Account) { + signer.storage.save(<-Test.createR(), to: /storage/r) + } + } + `), + }, + Context{ + Interface: runtimeInterface, + Location: nextTransactionLocation(), + }, + ) + require.NoError(t, err) + + assert.Equal(t, + expectedSaveToStorageTxWrites, + writes, + ) + + readTx := []byte(` + import Test from 0xCADE + + transaction { + + prepare(signer: auth(Storage) &Account) { + log(signer.storage.borrow<&Test.R>(from: /storage/r)!.test) + } + } + `) + + writes = nil + + err = runtime.ExecuteTransaction( + Script{ + Source: readTx, + }, + Context{ + Interface: runtimeInterface, + Location: nextTransactionLocation(), + }, + ) + require.NoError(t, err) + + assert.Empty(t, writes) + + writeTx := []byte(` + import Test from 0xCADE + + transaction { + + prepare(signer: auth(Storage) &Account) { + let r = signer.storage.borrow<&Test.R>(from: /storage/r)! + r.setTest(2) + } + } + `) + + writes = nil + + err = runtime.ExecuteTransaction( + Script{ + Source: writeTx, + }, + Context{ + Interface: runtimeInterface, + Location: nextTransactionLocation(), + }, + ) + require.NoError(t, err) + + assert.Equal(t, + expectedModifyStorageTxWrites, + writes, + ) }) } - runtimeInterface := &TestRuntimeInterface{ - OnGetCode: func(_ Location) (bytes []byte, err error) { - return accountCode, nil - }, - Storage: NewTestLedger(nil, onWrite), - OnGetSigningAccounts: func() ([]Address, error) { - return []Address{Address(addressValue)}, nil - }, - OnResolveLocation: NewSingleIdentifierLocationResolver(t), - OnGetAccountContractCode: func(location common.AddressLocation) (code []byte, err error) { - return accountCode, nil - }, - OnUpdateAccountContractCode: func(location common.AddressLocation, code []byte) error { - accountCode = code - return nil - }, - OnEmitEvent: func(event cadence.Event) error { - events = append(events, event) - return nil - }, - OnProgramLog: func(message string) { - loggedMessages = append(loggedMessages, message) + test( + false, + []ownerKeyPair{ + // storage index to contract domain storage map + { + addressValue[:], + []byte("contract"), + }, + // contract value + // NOTE: contract value slab is empty because it is inlined in contract domain storage map + { + addressValue[:], + []byte{'$', 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1}, + }, + // contract domain storage map + { + addressValue[:], + []byte{'$', 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2}, + }, }, - } - - nextTransactionLocation := NewTransactionLocationGenerator() - - err := runtime.ExecuteTransaction( - Script{ - Source: deploy, + []ownerKeyPair{ + // storage index to storage domain storage map + { + addressValue[:], + []byte("storage"), + }, + // resource value + // NOTE: resource value slab is empty because it is inlined in storage domain storage map + { + addressValue[:], + []byte{'$', 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3}, + }, + // storage domain storage map + // NOTE: resource value slab is inlined. + { + addressValue[:], + []byte{'$', 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x4}, + }, }, - Context{ - Interface: runtimeInterface, - Location: nextTransactionLocation(), + []ownerKeyPair{ + // Storage map is modified because resource value is inlined in storage map + // NOTE: resource value slab is empty. + { + addressValue[:], + []byte{'$', 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x4}, + }, }, ) - require.NoError(t, err) - - assert.NotNil(t, accountCode) - assert.Equal(t, + test( + true, []ownerKeyPair{ // storage index to account storage map { @@ -5959,32 +6171,7 @@ func TestRuntimeStorageWriteback(t *testing.T) { []byte{'$', 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3}, }, }, - writes, - ) - - writes = nil - - err = runtime.ExecuteTransaction( - Script{ - Source: []byte(` - import Test from 0xCADE - - transaction { - - prepare(signer: auth(Storage) &Account) { - signer.storage.save(<-Test.createR(), to: /storage/r) - } - } - `), - }, - Context{ - Interface: runtimeInterface, - Location: nextTransactionLocation(), - }, - ) - require.NoError(t, err) - assert.Equal(t, []ownerKeyPair{ // account storage map // NOTE: account storage map is updated with new storage domain storage map (inlined). @@ -6005,61 +6192,7 @@ func TestRuntimeStorageWriteback(t *testing.T) { []byte{'$', 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x5}, }, }, - writes, - ) - - readTx := []byte(` - import Test from 0xCADE - - transaction { - - prepare(signer: auth(Storage) &Account) { - log(signer.storage.borrow<&Test.R>(from: /storage/r)!.test) - } - } - `) - - writes = nil - err = runtime.ExecuteTransaction( - Script{ - Source: readTx, - }, - Context{ - Interface: runtimeInterface, - Location: nextTransactionLocation(), - }, - ) - require.NoError(t, err) - - assert.Empty(t, writes) - - writeTx := []byte(` - import Test from 0xCADE - - transaction { - - prepare(signer: auth(Storage) &Account) { - let r = signer.storage.borrow<&Test.R>(from: /storage/r)! - r.setTest(2) - } - } - `) - - writes = nil - - err = runtime.ExecuteTransaction( - Script{ - Source: writeTx, - }, - Context{ - Interface: runtimeInterface, - Location: nextTransactionLocation(), - }, - ) - require.NoError(t, err) - - assert.Equal(t, []ownerKeyPair{ // Account storage map is modified because resource value is inlined in storage map, // and storage map is inlined in account storage map. @@ -6069,7 +6202,6 @@ func TestRuntimeStorageWriteback(t *testing.T) { []byte{'$', 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2}, }, }, - writes, ) } @@ -7523,11 +7655,12 @@ func TestRuntimeComputationMetring(t *testing.T) { t.Parallel() type test struct { - name string - code string - ok bool - hits uint - intensity uint + name string + code string + ok bool + hits uint + v1Intensity uint + v2Intensity uint } compLimit := uint(6) @@ -7536,116 +7669,143 @@ func TestRuntimeComputationMetring(t *testing.T) { { name: "Infinite while loop", code: ` - while true {} - `, - ok: false, - hits: compLimit, - intensity: 6, + while true {} + `, + ok: false, + hits: compLimit, + v1Intensity: 6, + v2Intensity: 6, }, { name: "Limited while loop", code: ` - var i = 0 - while i < 5 { - i = i + 1 - } - `, - ok: false, - hits: compLimit, - intensity: 6, + var i = 0 + while i < 5 { + i = i + 1 + } + `, + ok: false, + hits: compLimit, + v1Intensity: 6, + v2Intensity: 6, }, { name: "statement + createArray + transferArray + too many for-in loop iterations", code: ` - for i in [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] {} - `, - ok: false, - hits: compLimit, - intensity: 6, + for i in [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] {} + `, + ok: false, + hits: compLimit, + v1Intensity: 6, + v2Intensity: 6, }, { name: "statement + createArray + transferArray + two for-in loop iterations", code: ` - for i in [1, 2] {} - `, - ok: true, - hits: 4, - intensity: 4, + for i in [1, 2] {} + `, + ok: true, + hits: 4, + v1Intensity: 4, + v2Intensity: 4, }, { name: "statement + functionInvocation + encoding", code: ` - acc.storage.save("A quick brown fox jumps over the lazy dog", to:/storage/some_path) - `, - ok: true, - hits: 3, - intensity: 108, + acc.storage.save("A quick brown fox jumps over the lazy dog", to:/storage/some_path) + `, + ok: true, + hits: 3, + v1Intensity: 76, + v2Intensity: 108, }, } - for _, test := range tests { + for _, testCase := range tests { - t.Run(test.name, func(t *testing.T) { + t.Run(testCase.name, func(t *testing.T) { - script := []byte( - fmt.Sprintf( - ` - transaction { - prepare(acc: auth(Storage) &Account) { - %s - } - } - `, - test.code, - ), - ) + test := func(storageFormatV2Enabled bool) { - runtime := NewTestInterpreterRuntime() + name := fmt.Sprintf( + "storage format V2 enabled: %v", + storageFormatV2Enabled, + ) + t.Run(name, func(t *testing.T) { + t.Parallel() - compErr := errors.New("computation exceeded limit") - var hits, totalIntensity uint - meterComputationFunc := func(kind common.ComputationKind, intensity uint) error { - hits++ - totalIntensity += intensity - if hits >= compLimit { - return compErr - } - return nil - } + script := []byte( + fmt.Sprintf( + ` + transaction { + prepare(acc: auth(Storage) &Account) { + %s + } + } + `, + testCase.code, + ), + ) + + config := DefaultTestInterpreterConfig + config.StorageFormatV2Enabled = storageFormatV2Enabled + runtime := NewTestInterpreterRuntimeWithConfig(config) + + compErr := errors.New("computation exceeded limit") + var hits, totalIntensity uint + meterComputationFunc := func(kind common.ComputationKind, intensity uint) error { + hits++ + totalIntensity += intensity + if hits >= compLimit { + return compErr + } + return nil + } - address := common.MustBytesToAddress([]byte{0x1}) + address := common.MustBytesToAddress([]byte{0x1}) - runtimeInterface := &TestRuntimeInterface{ - Storage: NewTestLedger(nil, nil), - OnGetSigningAccounts: func() ([]Address, error) { - return []Address{address}, nil - }, - OnMeterComputation: meterComputationFunc, - } + runtimeInterface := &TestRuntimeInterface{ + Storage: NewTestLedger(nil, nil), + OnGetSigningAccounts: func() ([]Address, error) { + return []Address{address}, nil + }, + OnMeterComputation: meterComputationFunc, + } - nextTransactionLocation := NewTransactionLocationGenerator() + nextTransactionLocation := NewTransactionLocationGenerator() - err := runtime.ExecuteTransaction( - Script{ - Source: script, - }, - Context{ - Interface: runtimeInterface, - Location: nextTransactionLocation(), - }, - ) - if test.ok { - require.NoError(t, err) - } else { - RequireError(t, err) + err := runtime.ExecuteTransaction( + Script{ + Source: script, + }, + Context{ + Interface: runtimeInterface, + Location: nextTransactionLocation(), + }, + ) + if testCase.ok { + require.NoError(t, err) + } else { + RequireError(t, err) + + var executionErr Error + require.ErrorAs(t, err, &executionErr) + require.ErrorAs(t, err.(Error).Unwrap(), &compErr) + } + + assert.Equal(t, testCase.hits, hits) - var executionErr Error - require.ErrorAs(t, err, &executionErr) - require.ErrorAs(t, err.(Error).Unwrap(), &compErr) + if storageFormatV2Enabled { + assert.Equal(t, testCase.v2Intensity, totalIntensity) + } else { + assert.Equal(t, testCase.v1Intensity, totalIntensity) + } + }) } - assert.Equal(t, test.hits, hits) - assert.Equal(t, test.intensity, totalIntensity) + for _, storageFormatV2Enabled := range []bool{false, true} { + test(storageFormatV2Enabled) + } }) } } diff --git a/runtime/script_executor.go b/runtime/script_executor.go index ca07c4cb00..8a51088a3d 100644 --- a/runtime/script_executor.go +++ b/runtime/script_executor.go @@ -107,7 +107,13 @@ func (executor *interpreterScriptExecutor) preprocess() (err error) { runtimeInterface := context.Interface - storage := NewStorage(runtimeInterface, runtimeInterface) + storage := NewStorage( + runtimeInterface, + runtimeInterface, + StorageConfig{ + StorageFormatV2Enabled: interpreterRuntime.defaultConfig.StorageFormatV2Enabled, + }, + ) executor.storage = storage environment := context.Environment diff --git a/runtime/sharedstate_test.go b/runtime/sharedstate_test.go index 287f03519c..52dd68f00a 100644 --- a/runtime/sharedstate_test.go +++ b/runtime/sharedstate_test.go @@ -19,6 +19,7 @@ package runtime_test import ( + "fmt" "testing" "github.com/stretchr/testify/assert" @@ -35,181 +36,224 @@ func TestRuntimeSharedState(t *testing.T) { t.Parallel() - runtime := NewTestInterpreterRuntime() - signerAddress := common.MustBytesToAddress([]byte{0x1}) - deploy1 := DeploymentTransaction("C1", []byte(` - access(all) contract C1 { - access(all) fun hello() { - log("Hello from C1!") - } - } - `)) + test := func( + storageFormatV2Enabled bool, + expectedReads []ownerKeyPair, + ) { - deploy2 := DeploymentTransaction("C2", []byte(` - access(all) contract C2 { - access(all) fun hello() { - log("Hello from C2!") - } - } - `)) + name := fmt.Sprintf( + "storage format V2 enabled: %v", + storageFormatV2Enabled, + ) - accountCodes := map[common.Location][]byte{} + t.Run(name, func(t *testing.T) { + t.Parallel() - var events []cadence.Event - var loggedMessages []string + config := DefaultTestInterpreterConfig + config.StorageFormatV2Enabled = storageFormatV2Enabled + config.AtreeValidationEnabled = false + runtime := NewTestInterpreterRuntimeWithConfig(config) - var interpreterState *interpreter.SharedState + deploy1 := DeploymentTransaction("C1", []byte(` + access(all) contract C1 { + access(all) fun hello() { + log("Hello from C1!") + } + } + `)) - var ledgerReads []ownerKeyPair + deploy2 := DeploymentTransaction("C2", []byte(` + access(all) contract C2 { + access(all) fun hello() { + log("Hello from C2!") + } + } + `)) - ledger := NewTestLedger( - func(owner, key, value []byte) { - ledgerReads = append( - ledgerReads, - ownerKeyPair{ - owner: owner, - key: key, - }, - ) - }, - nil, - ) + accountCodes := map[common.Location][]byte{} - runtimeInterface := &TestRuntimeInterface{ - Storage: ledger, - OnGetSigningAccounts: func() ([]Address, error) { - return []Address{signerAddress}, nil - }, - OnUpdateAccountContractCode: func(location common.AddressLocation, code []byte) error { - accountCodes[location] = code - return nil - }, - OnGetAccountContractCode: func(location common.AddressLocation) (code []byte, err error) { - code = accountCodes[location] - return code, nil - }, - OnRemoveAccountContractCode: func(location common.AddressLocation) error { - delete(accountCodes, location) - return nil - }, - OnResolveLocation: MultipleIdentifierLocationResolver, - OnProgramLog: func(message string) { - loggedMessages = append(loggedMessages, message) - }, - OnEmitEvent: func(event cadence.Event) error { - events = append(events, event) - return nil - }, - OnSetInterpreterSharedState: func(state *interpreter.SharedState) { - interpreterState = state - }, - OnGetInterpreterSharedState: func() *interpreter.SharedState { - return interpreterState - }, - } + var events []cadence.Event + var loggedMessages []string - environment := NewBaseInterpreterEnvironment(Config{}) + var interpreterState *interpreter.SharedState - nextTransactionLocation := NewTransactionLocationGenerator() + var ledgerReads []ownerKeyPair - // Deploy contracts + ledger := NewTestLedger( + func(owner, key, value []byte) { + ledgerReads = append( + ledgerReads, + ownerKeyPair{ + owner: owner, + key: key, + }, + ) + }, + nil, + ) - for _, source := range [][]byte{ - deploy1, - deploy2, - } { - err := runtime.ExecuteTransaction( - Script{ - Source: source, - }, - Context{ - Interface: runtimeInterface, - Location: nextTransactionLocation(), - Environment: environment, - }, - ) - require.NoError(t, err) - } + runtimeInterface := &TestRuntimeInterface{ + Storage: ledger, + OnGetSigningAccounts: func() ([]Address, error) { + return []Address{signerAddress}, nil + }, + OnUpdateAccountContractCode: func(location common.AddressLocation, code []byte) error { + accountCodes[location] = code + return nil + }, + OnGetAccountContractCode: func(location common.AddressLocation) (code []byte, err error) { + code = accountCodes[location] + return code, nil + }, + OnRemoveAccountContractCode: func(location common.AddressLocation) error { + delete(accountCodes, location) + return nil + }, + OnResolveLocation: MultipleIdentifierLocationResolver, + OnProgramLog: func(message string) { + loggedMessages = append(loggedMessages, message) + }, + OnEmitEvent: func(event cadence.Event) error { + events = append(events, event) + return nil + }, + OnSetInterpreterSharedState: func(state *interpreter.SharedState) { + interpreterState = state + }, + OnGetInterpreterSharedState: func() *interpreter.SharedState { + return interpreterState + }, + } + + environment := NewBaseInterpreterEnvironment(config) + + nextTransactionLocation := NewTransactionLocationGenerator() + + // Deploy contracts + + for _, source := range [][]byte{ + deploy1, + deploy2, + } { + err := runtime.ExecuteTransaction( + Script{ + Source: source, + }, + Context{ + Interface: runtimeInterface, + Location: nextTransactionLocation(), + Environment: environment, + }, + ) + require.NoError(t, err) + } + + assert.NotEmpty(t, accountCodes) + + // Call C1.hello using transaction + + loggedMessages = nil + + err := runtime.ExecuteTransaction( + Script{ + Source: []byte(` + import C1 from 0x1 + + transaction { + prepare(signer: &Account) { + C1.hello() + } + } + `), + Arguments: nil, + }, + Context{ + Interface: runtimeInterface, + Location: nextTransactionLocation(), + Environment: environment, + }, + ) + require.NoError(t, err) - assert.NotEmpty(t, accountCodes) + assert.Equal(t, []string{`"Hello from C1!"`}, loggedMessages) - // Call C1.hello using transaction + // Call C1.hello manually - loggedMessages = nil + loggedMessages = nil - err := runtime.ExecuteTransaction( - Script{ - Source: []byte(` - import C1 from 0x1 + _, err = runtime.InvokeContractFunction( + common.AddressLocation{ + Address: signerAddress, + Name: "C1", + }, + "hello", + nil, + nil, + Context{ + Interface: runtimeInterface, + Location: nextTransactionLocation(), + Environment: environment, + }, + ) + require.NoError(t, err) - transaction { - prepare(signer: &Account) { - C1.hello() - } - } - `), - Arguments: nil, - }, - Context{ - Interface: runtimeInterface, - Location: nextTransactionLocation(), - Environment: environment, - }, - ) - require.NoError(t, err) + assert.Equal(t, []string{`"Hello from C1!"`}, loggedMessages) - assert.Equal(t, []string{`"Hello from C1!"`}, loggedMessages) + // Call C2.hello manually - // Call C1.hello manually + loggedMessages = nil - loggedMessages = nil + _, err = runtime.InvokeContractFunction( + common.AddressLocation{ + Address: signerAddress, + Name: "C2", + }, + "hello", + nil, + nil, + Context{ + Interface: runtimeInterface, + Location: nextTransactionLocation(), + Environment: environment, + }, + ) + require.NoError(t, err) - _, err = runtime.InvokeContractFunction( - common.AddressLocation{ - Address: signerAddress, - Name: "C1", - }, - "hello", - nil, - nil, - Context{ - Interface: runtimeInterface, - Location: nextTransactionLocation(), - Environment: environment, - }, - ) - require.NoError(t, err) + assert.Equal(t, []string{`"Hello from C2!"`}, loggedMessages) - assert.Equal(t, []string{`"Hello from C1!"`}, loggedMessages) + // Assert shared state was used, + // i.e. data was not re-read - // Call C2.hello manually + require.Equal(t, + expectedReads, + ledgerReads, + ) + }) + } - loggedMessages = nil + test( + false, - _, err = runtime.InvokeContractFunction( - common.AddressLocation{ - Address: signerAddress, - Name: "C2", - }, - "hello", - nil, - nil, - Context{ - Interface: runtimeInterface, - Location: nextTransactionLocation(), - Environment: environment, + []ownerKeyPair{ + { + owner: signerAddress[:], + key: []byte(common.StorageDomainContract.Identifier()), + }, + { + owner: signerAddress[:], + key: []byte(common.StorageDomainContract.Identifier()), + }, + { + owner: signerAddress[:], + key: []byte{'$', 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2}, + }, }, ) - require.NoError(t, err) - assert.Equal(t, []string{`"Hello from C2!"`}, loggedMessages) - - // Assert shared state was used, - // i.e. data was not re-read - - require.Equal(t, + test( + true, []ownerKeyPair{ // Read account domain register to check if it is a migrated account // Read returns no value. @@ -217,12 +261,6 @@ func TestRuntimeSharedState(t *testing.T) { owner: signerAddress[:], key: []byte(AccountStorageKey), }, - // Read contract domain register to check if it is a unmigrated account - // Read returns no value. - { - owner: signerAddress[:], - key: []byte(common.StorageDomainContract.Identifier()), - }, // Read all available domain registers to check if it is a new account // Read returns no value. { @@ -261,17 +299,22 @@ func TestRuntimeSharedState(t *testing.T) { owner: signerAddress[:], key: []byte(common.StorageDomainAccountCapability.Identifier()), }, - // Read account domain register { owner: signerAddress[:], key: []byte(AccountStorageKey), }, - // Read account storage map + { + owner: signerAddress[:], + key: []byte(AccountStorageKey), + }, + { + owner: signerAddress[:], + key: []byte(AccountStorageKey), + }, { owner: signerAddress[:], key: []byte{'$', 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2}, }, }, - ledgerReads, ) } diff --git a/runtime/slabindex.go b/runtime/slabindex.go new file mode 100644 index 0000000000..00178608d0 --- /dev/null +++ b/runtime/slabindex.go @@ -0,0 +1,86 @@ +/* + * Cadence - The resource-oriented smart contract programming language + * + * Copyright Flow Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package runtime + +import ( + "github.com/onflow/atree" + + "github.com/onflow/cadence/common" + "github.com/onflow/cadence/errors" + "github.com/onflow/cadence/interpreter" +) + +// readSlabIndexFromRegister returns register value as atree.SlabIndex. +// This function returns error if +// - underlying ledger panics, or +// - underlying ledger returns error when retrieving ledger value, or +// - retrieved ledger value is invalid (for atree.SlabIndex). +func readSlabIndexFromRegister( + ledger atree.Ledger, + address common.Address, + key []byte, +) (atree.SlabIndex, bool, error) { + var data []byte + var err error + errors.WrapPanic(func() { + data, err = ledger.GetValue(address[:], key) + }) + if err != nil { + return atree.SlabIndex{}, false, interpreter.WrappedExternalError(err) + } + + dataLength := len(data) + + if dataLength == 0 { + return atree.SlabIndex{}, false, nil + } + + isStorageIndex := dataLength == storageIndexLength + if !isStorageIndex { + // Invalid data in register + + // TODO: add dedicated error type? + return atree.SlabIndex{}, false, errors.NewUnexpectedError( + "invalid storage index for storage map of account '%x': expected length %d, got %d", + address[:], storageIndexLength, dataLength, + ) + } + + return atree.SlabIndex(data), true, nil +} + +func writeSlabIndexToRegister( + ledger atree.Ledger, + address common.Address, + key []byte, + slabIndex atree.SlabIndex, +) error { + var err error + errors.WrapPanic(func() { + err = ledger.SetValue( + address[:], + key, + slabIndex[:], + ) + }) + if err != nil { + return interpreter.WrappedExternalError(err) + } + return nil +} diff --git a/runtime/storage.go b/runtime/storage.go index 253aeabe60..8198e69fa1 100644 --- a/runtime/storage.go +++ b/runtime/storage.go @@ -36,25 +36,21 @@ const ( AccountStorageKey = "stored" ) +type StorageConfig struct { + StorageFormatV2Enabled bool +} + type Storage struct { *atree.PersistentSlabStorage - // NewAccountStorageMapSlabIndices contains root slab index of new accounts' storage map. - // The indices are saved using Ledger.SetValue() during Commit(). - // Key is StorageKey{address, accountStorageKey} and value is 8-byte slab index. - NewAccountStorageMapSlabIndices *orderedmap.OrderedMap[interpreter.StorageKey, atree.SlabIndex] - - // unmigratedAccounts are accounts that were accessed but not migrated. - unmigratedAccounts *orderedmap.OrderedMap[common.Address, struct{}] - - // cachedAccountStorageMaps is a cache of account storage maps. - // Key is StorageKey{address, accountStorageKey} and value is account storage map. - cachedAccountStorageMaps map[interpreter.StorageKey]*interpreter.AccountStorageMap - // cachedDomainStorageMaps is a cache of domain storage maps. // Key is StorageKey{address, domain} and value is domain storage map. cachedDomainStorageMaps map[interpreter.StorageDomainKey]*interpreter.DomainStorageMap + // cachedV1Accounts contains the cached result of determining + // if the account is in storage format v1 or not. + cachedV1Accounts map[common.Address]bool + // contractUpdates is a cache of contract updates. // Key is StorageKey{contract_address, contract_name} and value is contract composite value. contractUpdates *orderedmap.OrderedMap[interpreter.StorageKey, *interpreter.CompositeValue] @@ -62,12 +58,22 @@ type Storage struct { Ledger atree.Ledger memoryGauge common.MemoryGauge + + Config StorageConfig + + AccountStorageV1 *AccountStorageV1 + AccountStorageV2 *AccountStorageV2 + scheduledV2Migrations []common.Address } var _ atree.SlabStorage = &Storage{} var _ interpreter.Storage = &Storage{} -func NewStorage(ledger atree.Ledger, memoryGauge common.MemoryGauge) *Storage { +func NewStorage( + ledger atree.Ledger, + memoryGauge common.MemoryGauge, + config StorageConfig, +) *Storage { decodeStorable := func( decoder *cbor.StreamDecoder, slabID atree.SlabID, @@ -96,318 +102,149 @@ func NewStorage(ledger atree.Ledger, memoryGauge common.MemoryGauge) *Storage { decodeStorable, decodeTypeInfo, ) + + accountStorageV1 := NewAccountStorageV1( + ledger, + persistentSlabStorage, + memoryGauge, + ) + + var accountStorageV2 *AccountStorageV2 + if config.StorageFormatV2Enabled { + accountStorageV2 = NewAccountStorageV2( + ledger, + persistentSlabStorage, + memoryGauge, + ) + } + return &Storage{ - Ledger: ledger, - PersistentSlabStorage: persistentSlabStorage, - cachedAccountStorageMaps: map[interpreter.StorageKey]*interpreter.AccountStorageMap{}, - cachedDomainStorageMaps: map[interpreter.StorageDomainKey]*interpreter.DomainStorageMap{}, - memoryGauge: memoryGauge, + Ledger: ledger, + PersistentSlabStorage: persistentSlabStorage, + memoryGauge: memoryGauge, + Config: config, + AccountStorageV1: accountStorageV1, + AccountStorageV2: accountStorageV2, } } const storageIndexLength = 8 -// GetStorageMap returns existing or new domain storage map for the given account and domain. -func (s *Storage) GetStorageMap( +// GetDomainStorageMap returns existing or new domain storage map for the given account and domain. +func (s *Storage) GetDomainStorageMap( inter *interpreter.Interpreter, address common.Address, domain common.StorageDomain, createIfNotExists bool, ) ( - storageMap *interpreter.DomainStorageMap, + domainStorageMap *interpreter.DomainStorageMap, ) { - - // Account can be migrated account, new account, or unmigrated account. - // - // - // ### Migrated Account - // - // Migrated account is account with AccountStorageKey register. - // Migrated account has account storage map, which contains domain storage maps - // with domain as key. - // - // If domain exists in the account storage map, domain storage map is returned. - // - // If domain doesn't exist and createIfNotExists is true, - // new domain storage map is created, inserted into account storage map, and returned. - // - // If domain doesn't exist and createIfNotExists is false, nil is returned. - // - // - // ### New Account - // - // New account is account without AccountStorageKey register and without any domain registers. - // NOTE: new account's AccountStorageKey register is persisted in Commit(). - // - // If createIfNotExists is true, - // - new account storage map is created - // - new domain storage map is created - // - domain storage map is inserted into account storage map - // - domain storage map is returned - // - // If createIfNotExists is false, nil is returned. - // - // - // ### Unmigrated Account - // - // Unmigrated account is account with at least one domain register. - // Unmigrated account has domain registers and corresponding domain storage maps. - // - // If domain exists (domain register exists), domain storage map is loaded and returned. - // - // If domain doesn't exist and createIfNotExists is true, - // new domain storage map is created and returned. - // NOTE: given account would be migrated in Commit() since this is write op. - // - // If domain doesn't exist and createIfNotExists is false, nil is returned. - // - // - // ### Migration of unmigrated accounts - // - // Migration happens in Commit() if unmigrated account has write ops. - // NOTE: Commit() is not called by this function. - // - // Specifically, - // - unmigrated account is migrated in Commit() if there are write ops. - // For example, inserting values in unmigrated account triggers migration in Commit(). - // - unmigrated account is unchanged if there are only read ops. - // For example, iterating values in unmigrated account doesn't trigger migration, - // and checking if domain exists doesn't trigger migration. - // Get cached domain storage map if it exists. domainStorageKey := interpreter.NewStorageDomainKey(s.memoryGauge, address, domain) - if domainStorageMap := s.cachedDomainStorageMaps[domainStorageKey]; domainStorageMap != nil { - return domainStorageMap - } - - // Get (or create) domain storage map from existing account storage map - // if account is migrated account. - - accountStorageKey := interpreter.NewStorageKey(s.memoryGauge, address, AccountStorageKey) - - accountStorageMap, err := s.getAccountStorageMap(accountStorageKey) - if err != nil { - panic(err) - } - - if accountStorageMap != nil { - // This is migrated account. - - // Get (or create) domain storage map from account storage map. - domainStorageMap := accountStorageMap.GetDomain(s.memoryGauge, inter, domain, createIfNotExists) - - // Cache domain storage map + if s.cachedDomainStorageMaps != nil { + domainStorageMap = s.cachedDomainStorageMaps[domainStorageKey] if domainStorageMap != nil { - s.cachedDomainStorageMaps[domainStorageKey] = domainStorageMap + return domainStorageMap } - - return domainStorageMap } - // At this point, account is either new or unmigrated account. - - domainStorageMap, err := getDomainStorageMapFromLegacyDomainRegister(s.Ledger, s.PersistentSlabStorage, address, domain) - if err != nil { - panic(err) - } - - if domainStorageMap != nil { - // This is a unmigrated account with given domain register. - + defer func() { // Cache domain storage map - s.cachedDomainStorageMaps[domainStorageKey] = domainStorageMap - - // Add account to unmigrated account list - s.addUnmigratedAccount(address) - - return domainStorageMap - } - - // At this point, account is either new account or unmigrated account without given domain. - - // Domain doesn't exist. Return early if createIfNotExists is false. - - if !createIfNotExists { - return nil - } - - // Handle unmigrated account - unmigrated, err := s.isUnmigratedAccount(address) - if err != nil { - panic(err) - } - if unmigrated { - // Add account to unmigrated account list - s.addUnmigratedAccount(address) + if domainStorageMap != nil { + s.cacheDomainStorageMap( + domainStorageKey, + domainStorageMap, + ) + } + }() - // Create new domain storage map - domainStorageMap := interpreter.NewDomainStorageMap(s.memoryGauge, s, atree.Address(address)) + if !s.Config.StorageFormatV2Enabled || s.IsV1Account(address) { + domainStorageMap = s.AccountStorageV1.GetDomainStorageMap( + address, + domain, + createIfNotExists, + ) - // Cache new domain storage map - s.cachedDomainStorageMaps[domainStorageKey] = domainStorageMap + if domainStorageMap != nil { + s.cacheIsV1Account(address, true) + } - return domainStorageMap + } else { + domainStorageMap = s.AccountStorageV2.GetDomainStorageMap( + inter, + address, + domain, + createIfNotExists, + ) } - // Handle new account - - // Create account storage map - accountStorageMap = interpreter.NewAccountStorageMap(s.memoryGauge, s, atree.Address(address)) - - // Cache account storage map - s.cachedAccountStorageMaps[accountStorageKey] = accountStorageMap - - // Create new domain storage map as an element in account storage map - domainStorageMap = accountStorageMap.NewDomain(s.memoryGauge, inter, domain) - - // Cache domain storage map - s.cachedDomainStorageMaps[domainStorageKey] = domainStorageMap - - // Save new account and its account storage map root SlabID to new accout list - s.addNewAccount(accountStorageKey, accountStorageMap.SlabID().Index()) - return domainStorageMap } -// getAccountStorageMap returns AccountStorageMap if exists, or nil otherwise. -func (s *Storage) getAccountStorageMap(accountStorageKey interpreter.StorageKey) (*interpreter.AccountStorageMap, error) { +// IsV1Account returns true if given account is in account storage format v1. +func (s *Storage) IsV1Account(address common.Address) (isV1 bool) { - // Return cached account storage map if available. + // Check cache - accountStorageMap := s.cachedAccountStorageMaps[accountStorageKey] - if accountStorageMap != nil { - return accountStorageMap, nil + if isV1, present := s.cachedV1Accounts[address]; present { + return isV1 } - // Load account storage map if account storage register exists. + // Cache result - accountStorageSlabIndex, accountStorageRegisterExists, err := getSlabIndexFromRegisterValue( - s.Ledger, - accountStorageKey.Address, - []byte(accountStorageKey.Key), - ) - if err != nil { - return nil, err - } - if !accountStorageRegisterExists { - return nil, nil - } - - slabID := atree.NewSlabID( - atree.Address(accountStorageKey.Address), - accountStorageSlabIndex, - ) - - accountStorageMap = interpreter.NewAccountStorageMapWithRootID(s, slabID) - - // Cache account storage map + defer func() { + s.cacheIsV1Account(address, isV1) + }() - s.cachedAccountStorageMaps[accountStorageKey] = accountStorageMap + // First check if account storage map exists. + // In that case the account was already migrated to account storage format v2, + // and we do not need to check the domain storage map registers. - return accountStorageMap, nil -} - -// getDomainStorageMapFromLegacyDomainRegister returns domain storage map from legacy domain register. -func getDomainStorageMapFromLegacyDomainRegister( - ledger atree.Ledger, - storage atree.SlabStorage, - address common.Address, - domain common.StorageDomain, -) (*interpreter.DomainStorageMap, error) { - domainStorageSlabIndex, domainRegisterExists, err := getSlabIndexFromRegisterValue( - ledger, - address, - []byte(domain.Identifier())) + accountStorageMapExists, err := hasAccountStorageMap(s.Ledger, address) if err != nil { - return nil, err - } - if !domainRegisterExists { - return nil, nil - } - - slabID := atree.NewSlabID(atree.Address(address), domainStorageSlabIndex) - return interpreter.NewDomainStorageMapWithRootID(storage, slabID), nil -} - -func (s *Storage) addUnmigratedAccount(address common.Address) { - if s.unmigratedAccounts == nil { - s.unmigratedAccounts = &orderedmap.OrderedMap[common.Address, struct{}]{} - } - if !s.unmigratedAccounts.Contains(address) { - s.unmigratedAccounts.Set(address, struct{}{}) - } -} - -func (s *Storage) addNewAccount(accountStorageKey interpreter.StorageKey, slabIndex atree.SlabIndex) { - if s.NewAccountStorageMapSlabIndices == nil { - s.NewAccountStorageMapSlabIndices = &orderedmap.OrderedMap[interpreter.StorageKey, atree.SlabIndex]{} + panic(err) } - s.NewAccountStorageMapSlabIndices.Set(accountStorageKey, slabIndex) -} - -// isUnmigratedAccount returns true if given account has any domain registers. -func (s *Storage) isUnmigratedAccount(address common.Address) (bool, error) { - if s.unmigratedAccounts != nil && - s.unmigratedAccounts.Contains(address) { - return true, nil + if accountStorageMapExists { + return false } - // Check most frequently used domains first, such as storage, public, private. + // Check if a storage map register exists for any of the domains. + // Check the most frequently used domains first, such as storage, public, private. for _, domain := range common.AllStorageDomains { - _, domainExists, err := getSlabIndexFromRegisterValue( + _, domainExists, err := readSlabIndexFromRegister( s.Ledger, address, - []byte(domain.Identifier())) + []byte(domain.Identifier()), + ) if err != nil { - return false, err + panic(err) } if domainExists { - return true, nil + return true } } - return false, nil + return false } -// getSlabIndexFromRegisterValue returns register value as atree.SlabIndex. -// This function returns error if -// - underlying ledger panics, or -// - underlying ledger returns error when retrieving ledger value, or -// - retrieved ledger value is invalid (for atree.SlabIndex). -func getSlabIndexFromRegisterValue( - ledger atree.Ledger, - address common.Address, - key []byte, -) (atree.SlabIndex, bool, error) { - var data []byte - var err error - errors.WrapPanic(func() { - data, err = ledger.GetValue(address[:], key) - }) - if err != nil { - return atree.SlabIndex{}, false, interpreter.WrappedExternalError(err) +func (s *Storage) cacheIsV1Account(address common.Address, isV1 bool) { + if s.cachedV1Accounts == nil { + s.cachedV1Accounts = map[common.Address]bool{} } + s.cachedV1Accounts[address] = isV1 +} - dataLength := len(data) - - if dataLength == 0 { - return atree.SlabIndex{}, false, nil - } - - isStorageIndex := dataLength == storageIndexLength - if !isStorageIndex { - // Invalid data in register - - // TODO: add dedicated error type? - return atree.SlabIndex{}, false, errors.NewUnexpectedError( - "invalid storage index for storage map of account '%x': expected length %d, got %d", - address[:], storageIndexLength, dataLength, - ) +func (s *Storage) cacheDomainStorageMap( + storageDomainKey interpreter.StorageDomainKey, + domainStorageMap *interpreter.DomainStorageMap, +) { + if s.cachedDomainStorageMaps == nil { + s.cachedDomainStorageMaps = map[interpreter.StorageDomainKey]*interpreter.DomainStorageMap{} } - return atree.SlabIndex(data), true, nil + s.cachedDomainStorageMaps[storageDomainKey] = domainStorageMap } func (s *Storage) recordContractUpdate( @@ -466,7 +303,7 @@ func (s *Storage) writeContractUpdate( key interpreter.StorageKey, contractValue *interpreter.CompositeValue, ) { - storageMap := s.GetStorageMap(inter, key.Address, common.StorageDomainContract, true) + storageMap := s.GetDomainStorageMap(inter, key.Address, common.StorageDomainContract, true) // NOTE: pass nil instead of allocating a Value-typed interface that points to nil storageMapKey := interpreter.StringStorageMapKey(key.Key) if contractValue == nil { @@ -481,7 +318,7 @@ func (s *Storage) Commit(inter *interpreter.Interpreter, commitContractUpdates b return s.commit(inter, commitContractUpdates, true) } -// NondeterministicCommit serializes and commits all values in the deltas storage +// Deprecated: NondeterministicCommit serializes and commits all values in the deltas storage // in nondeterministic order. This function is used when commit ordering isn't // required (e.g. migration programs). func (s *Storage) NondeterministicCommit(inter *interpreter.Interpreter, commitContractUpdates bool) error { @@ -494,70 +331,65 @@ func (s *Storage) commit(inter *interpreter.Interpreter, commitContractUpdates b s.commitContractUpdates(inter) } - err := s.commitNewStorageMaps() + err := s.AccountStorageV1.commit() if err != nil { return err } - // Migrate accounts that have write ops before calling PersistentSlabStorage.FastCommit(). - err = s.migrateAccountsIfNeeded(inter) - if err != nil { - return err + if s.Config.StorageFormatV2Enabled { + err = s.AccountStorageV2.commit() + if err != nil { + return err + } + + err = s.migrateV1AccountsToV2(inter) + if err != nil { + return err + } } // Commit the underlying slab storage's writes - size := s.PersistentSlabStorage.DeltasSizeWithoutTempAddresses() + slabStorage := s.PersistentSlabStorage + + size := slabStorage.DeltasSizeWithoutTempAddresses() if size > 0 { inter.ReportComputation(common.ComputationKindEncodeValue, uint(size)) usage := common.NewBytesMemoryUsage(int(size)) - common.UseMemory(s.memoryGauge, usage) + common.UseMemory(inter, usage) } - deltas := s.PersistentSlabStorage.DeltasWithoutTempAddresses() - common.UseMemory(s.memoryGauge, common.NewAtreeEncodedSlabMemoryUsage(deltas)) + deltas := slabStorage.DeltasWithoutTempAddresses() + common.UseMemory(inter, common.NewAtreeEncodedSlabMemoryUsage(deltas)) // TODO: report encoding metric for all encoded slabs if deterministic { - return s.PersistentSlabStorage.FastCommit(runtime.NumCPU()) + return slabStorage.FastCommit(runtime.NumCPU()) } else { - return s.PersistentSlabStorage.NondeterministicFastCommit(runtime.NumCPU()) + return slabStorage.NondeterministicFastCommit(runtime.NumCPU()) } } -func (s *Storage) commitNewStorageMaps() error { - if s.NewAccountStorageMapSlabIndices == nil { - return nil - } +func (s *Storage) ScheduleV2Migration(address common.Address) { + s.scheduledV2Migrations = append(s.scheduledV2Migrations, address) +} - for pair := s.NewAccountStorageMapSlabIndices.Oldest(); pair != nil; pair = pair.Next() { - var err error - errors.WrapPanic(func() { - err = s.Ledger.SetValue( - pair.Key.Address[:], - []byte(pair.Key.Key), - pair.Value[:], - ) - }) - if err != nil { - return interpreter.WrappedExternalError(err) +func (s *Storage) ScheduleV2MigrationForModifiedAccounts() { + for address, isV1 := range s.cachedV1Accounts { //nolint:maprange + if isV1 && s.PersistentSlabStorage.HasUnsavedChanges(atree.Address(address)) { + s.ScheduleV2Migration(address) } } - - return nil } -func (s *Storage) migrateAccountsIfNeeded(inter *interpreter.Interpreter) error { - if s.unmigratedAccounts == nil || s.unmigratedAccounts.Len() == 0 { - return nil +func (s *Storage) migrateV1AccountsToV2(inter *interpreter.Interpreter) error { + + if !s.Config.StorageFormatV2Enabled { + return errors.NewUnexpectedError("cannot migrate to storage format v2, as it is not enabled") } - return s.migrateAccounts(inter) -} -func (s *Storage) migrateAccounts(inter *interpreter.Interpreter) error { - // Predicate function allows migration for accounts with write ops. - migrateAccountPred := func(address common.Address) bool { - return s.PersistentSlabStorage.HasUnsavedChanges(atree.Address(address)) + if len(s.scheduledV2Migrations) == 0 { + return nil } // getDomainStorageMap function returns cached domain storage map if it is available @@ -578,38 +410,50 @@ func (s *Storage) migrateAccounts(inter *interpreter.Interpreter) error { return domainStorageMap, nil } - return getDomainStorageMapFromLegacyDomainRegister(ledger, storage, address, domain) + return getDomainStorageMapFromV1DomainRegister(ledger, storage, address, domain) } - migrator := NewDomainRegisterMigration(s.Ledger, s.PersistentSlabStorage, inter, s.memoryGauge) - migrator.SetGetDomainStorageMapFunc(getDomainStorageMap) + migrator := NewDomainRegisterMigration( + s.Ledger, + s.PersistentSlabStorage, + inter, + s.memoryGauge, + getDomainStorageMap, + ) - migratedAccounts, err := migrator.MigrateAccounts(s.unmigratedAccounts, migrateAccountPred) - if err != nil { - return err - } + // Ensure the scheduled accounts are migrated in a deterministic order - if migratedAccounts == nil { - return nil - } + sort.Slice( + s.scheduledV2Migrations, + func(i, j int) bool { + address1 := s.scheduledV2Migrations[i] + address2 := s.scheduledV2Migrations[j] + return address1.Compare(address2) < 0 + }, + ) - // Update internal state with migrated accounts - for pair := migratedAccounts.Oldest(); pair != nil; pair = pair.Next() { - address := pair.Key - accountStorageMap := pair.Value + for _, address := range s.scheduledV2Migrations { - // Cache migrated account storage map - accountStorageKey := interpreter.NewStorageKey(s.memoryGauge, address, AccountStorageKey) - s.cachedAccountStorageMaps[accountStorageKey] = accountStorageMap + accountStorageMap, err := migrator.MigrateAccount(address) + if err != nil { + return err + } - // Remove migrated accounts from unmigratedAccounts - s.unmigratedAccounts.Delete(address) + s.AccountStorageV2.cacheAccountStorageMap( + address, + accountStorageMap, + ) + + s.cacheIsV1Account(address, false) } + s.scheduledV2Migrations = nil + return nil } func (s *Storage) CheckHealth() error { + // Check slab storage health rootSlabIDs, err := atree.CheckStorageHealth(s, -1) if err != nil { @@ -634,46 +478,47 @@ func (s *Storage) CheckHealth() error { var storageMapStorageIDs []atree.SlabID - // Get cached account storage map slab IDs. - for _, storageMap := range s.cachedAccountStorageMaps { //nolint:maprange + if s.Config.StorageFormatV2Enabled { + // Get cached account storage map slab IDs. storageMapStorageIDs = append( storageMapStorageIDs, - storageMap.SlabID(), + s.AccountStorageV2.cachedRootSlabIDs()..., ) } - // Get cached unmigrated domain storage map slab IDs + // Get slab IDs of cached domain storage maps that are in account storage format v1. for storageKey, storageMap := range s.cachedDomainStorageMaps { //nolint:maprange address := storageKey.Address - if s.unmigratedAccounts != nil && - s.unmigratedAccounts.Contains(address) { - - domainValueID := storageMap.ValueID() - - slabID := atree.NewSlabID( - atree.Address(address), - atree.SlabIndex(domainValueID[8:]), - ) - - storageMapStorageIDs = append( - storageMapStorageIDs, - slabID, - ) + // Only accounts in storage format v1 store domain storage maps + // directly at the root of the account + if !s.IsV1Account(address) { + continue } + + storageMapStorageIDs = append( + storageMapStorageIDs, + storageMap.SlabID(), + ) } - sort.Slice(storageMapStorageIDs, func(i, j int) bool { - a := storageMapStorageIDs[i] - b := storageMapStorageIDs[j] - return a.Compare(b) < 0 - }) + sort.Slice( + storageMapStorageIDs, + func(i, j int) bool { + a := storageMapStorageIDs[i] + b := storageMapStorageIDs[j] + return a.Compare(b) < 0 + }, + ) found := map[atree.SlabID]struct{}{} for _, storageMapStorageID := range storageMapStorageIDs { if _, ok := accountRootSlabIDs[storageMapStorageID]; !ok { - return errors.NewUnexpectedError("account storage map (and unmigrated domain storage map) points to non-root slab %s", storageMapStorageID) + return errors.NewUnexpectedError( + "account storage map (and unmigrated domain storage map) points to non-root slab %s", + storageMapStorageID, + ) } found[storageMapStorageID] = struct{}{} diff --git a/runtime/storage_test.go b/runtime/storage_test.go index 5c1299df9f..4e4dc261bc 100644 --- a/runtime/storage_test.go +++ b/runtime/storage_test.go @@ -35,7 +35,6 @@ import ( "github.com/onflow/cadence" "github.com/onflow/cadence/common" - "github.com/onflow/cadence/common/orderedmap" "github.com/onflow/cadence/encoding/json" "github.com/onflow/cadence/interpreter" . "github.com/onflow/cadence/runtime" @@ -52,7 +51,13 @@ func withWritesToStorage( handler func(*Storage, *interpreter.Interpreter), ) { ledger := NewTestLedger(nil, onWrite) - storage := NewStorage(ledger, nil) + storage := NewStorage( + ledger, + nil, + StorageConfig{ + StorageFormatV2Enabled: true, + }, + ) inter := NewTestInterpreter(tb) @@ -63,18 +68,10 @@ func withWritesToStorage( var address common.Address random.Read(address[:]) - storageKey := interpreter.StorageKey{ - Address: address, - Key: AccountStorageKey, - } - var slabIndex atree.SlabIndex binary.BigEndian.PutUint32(slabIndex[:], randomIndex) - if storage.NewAccountStorageMapSlabIndices == nil { - storage.NewAccountStorageMapSlabIndices = &orderedmap.OrderedMap[interpreter.StorageKey, atree.SlabIndex]{} - } - storage.NewAccountStorageMapSlabIndices.Set(storageKey, slabIndex) + storage.AccountStorageV2.SetNewAccountStorageMapSlabIndex(address, slabIndex) } handler(storage, inter) @@ -158,7 +155,9 @@ func TestRuntimeStorageWrite(t *testing.T) { t.Parallel() - runtime := NewTestInterpreterRuntime() + config := DefaultTestInterpreterConfig + config.StorageFormatV2Enabled = true + runtime := NewTestInterpreterRuntimeWithConfig(config) address := common.MustBytesToAddress([]byte{0x1}) @@ -1619,161 +1618,238 @@ func TestRuntimeResourceOwnerChange(t *testing.T) { t.Parallel() - config := DefaultTestInterpreterConfig - config.ResourceOwnerChangeHandlerEnabled = true - runtime := NewTestInterpreterRuntimeWithConfig(config) + test := func( + storageFormatV2Enabled bool, + expectedNonEmptyKeys []string, + ) { - address1 := common.MustBytesToAddress([]byte{0x1}) - address2 := common.MustBytesToAddress([]byte{0x2}) + name := fmt.Sprintf( + "storage format V2 enabled: %v", + storageFormatV2Enabled, + ) + t.Run(name, func(t *testing.T) { + t.Parallel() - ledger := NewTestLedger(nil, nil) + config := DefaultTestInterpreterConfig + config.ResourceOwnerChangeHandlerEnabled = true + config.StorageFormatV2Enabled = storageFormatV2Enabled + runtime := NewTestInterpreterRuntimeWithConfig(config) - var signers []Address + address1 := common.MustBytesToAddress([]byte{0x1}) + address2 := common.MustBytesToAddress([]byte{0x2}) - deployTx := DeploymentTransaction("Test", []byte(` - access(all) contract Test { + ledger := NewTestLedger(nil, nil) - access(all) resource R {} + var signers []Address - access(all) fun createR(): @R { - return <-create R() - } - } - `)) + deployTx := DeploymentTransaction("Test", []byte(` + access(all) contract Test { - type resourceOwnerChange struct { - uuid *interpreter.UInt64Value - typeID common.TypeID - oldAddress common.Address - newAddress common.Address - } + access(all) resource R {} - accountCodes := map[Location][]byte{} - var events []cadence.Event - var loggedMessages []string - var resourceOwnerChanges []resourceOwnerChange + access(all) fun createR(): @R { + return <-create R() + } + } + `)) - runtimeInterface := &TestRuntimeInterface{ - Storage: ledger, - OnGetSigningAccounts: func() ([]Address, error) { - return signers, nil - }, - OnResolveLocation: NewSingleIdentifierLocationResolver(t), - OnUpdateAccountContractCode: func(location common.AddressLocation, code []byte) error { - accountCodes[location] = code - return nil - }, - OnGetAccountContractCode: func(location common.AddressLocation) (code []byte, err error) { - code = accountCodes[location] - return code, nil - }, - OnEmitEvent: func(event cadence.Event) error { - events = append(events, event) - return nil - }, - OnProgramLog: func(message string) { - loggedMessages = append(loggedMessages, message) - }, - OnResourceOwnerChanged: func( - inter *interpreter.Interpreter, - resource *interpreter.CompositeValue, - oldAddress common.Address, - newAddress common.Address, - ) { - resourceOwnerChanges = append( - resourceOwnerChanges, - resourceOwnerChange{ - typeID: resource.TypeID(), - // TODO: provide proper location range - uuid: resource.ResourceUUID(inter, interpreter.EmptyLocationRange), - oldAddress: oldAddress, - newAddress: newAddress, + type resourceOwnerChange struct { + uuid *interpreter.UInt64Value + typeID common.TypeID + oldAddress common.Address + newAddress common.Address + } + + accountCodes := map[Location][]byte{} + var events []cadence.Event + var loggedMessages []string + var resourceOwnerChanges []resourceOwnerChange + + runtimeInterface := &TestRuntimeInterface{ + Storage: ledger, + OnGetSigningAccounts: func() ([]Address, error) { + return signers, nil }, - ) - }, - } + OnResolveLocation: NewSingleIdentifierLocationResolver(t), + OnUpdateAccountContractCode: func(location common.AddressLocation, code []byte) error { + accountCodes[location] = code + return nil + }, + OnGetAccountContractCode: func(location common.AddressLocation) (code []byte, err error) { + code = accountCodes[location] + return code, nil + }, + OnEmitEvent: func(event cadence.Event) error { + events = append(events, event) + return nil + }, + OnProgramLog: func(message string) { + loggedMessages = append(loggedMessages, message) + }, + OnResourceOwnerChanged: func( + inter *interpreter.Interpreter, + resource *interpreter.CompositeValue, + oldAddress common.Address, + newAddress common.Address, + ) { + resourceOwnerChanges = append( + resourceOwnerChanges, + resourceOwnerChange{ + typeID: resource.TypeID(), + // TODO: provide proper location range + uuid: resource.ResourceUUID(inter, interpreter.EmptyLocationRange), + oldAddress: oldAddress, + newAddress: newAddress, + }, + ) + }, + } - nextTransactionLocation := NewTransactionLocationGenerator() + nextTransactionLocation := NewTransactionLocationGenerator() - // Deploy contract + // Deploy contract - signers = []Address{address1} + signers = []Address{address1} - err := runtime.ExecuteTransaction( - Script{ - Source: deployTx, - }, - Context{ - Interface: runtimeInterface, - Location: nextTransactionLocation(), - }, - ) - require.NoError(t, err) + err := runtime.ExecuteTransaction( + Script{ + Source: deployTx, + }, + Context{ + Interface: runtimeInterface, + Location: nextTransactionLocation(), + }, + ) + require.NoError(t, err) - // Store + // Store - signers = []Address{address1} + signers = []Address{address1} - storeTx := []byte(` - import Test from 0x1 + storeTx := []byte(` + import Test from 0x1 - transaction { - prepare(signer: auth(Storage) &Account) { - signer.storage.save(<-Test.createR(), to: /storage/test) - } - } - `) + transaction { + prepare(signer: auth(Storage) &Account) { + signer.storage.save(<-Test.createR(), to: /storage/test) + } + } + `) - err = runtime.ExecuteTransaction( - Script{ - Source: storeTx, - }, - Context{ - Interface: runtimeInterface, - Location: nextTransactionLocation(), - }, - ) - require.NoError(t, err) + err = runtime.ExecuteTransaction( + Script{ + Source: storeTx, + }, + Context{ + Interface: runtimeInterface, + Location: nextTransactionLocation(), + }, + ) + require.NoError(t, err) - // Transfer + // Transfer - signers = []Address{address1, address2} + signers = []Address{address1, address2} - transferTx := []byte(` - import Test from 0x1 + transferTx := []byte(` + import Test from 0x1 - transaction { - prepare( - signer1: auth(Storage) &Account, - signer2: auth(Storage) &Account - ) { - let value <- signer1.storage.load<@Test.R>(from: /storage/test)! - signer2.storage.save(<-value, to: /storage/test) - } - } - `) + transaction { + prepare( + signer1: auth(Storage) &Account, + signer2: auth(Storage) &Account + ) { + let value <- signer1.storage.load<@Test.R>(from: /storage/test)! + signer2.storage.save(<-value, to: /storage/test) + } + } + `) - err = runtime.ExecuteTransaction( - Script{ - Source: transferTx, - }, - Context{ - Interface: runtimeInterface, - Location: nextTransactionLocation(), - }, - ) - require.NoError(t, err) + err = runtime.ExecuteTransaction( + Script{ + Source: transferTx, + }, + Context{ + Interface: runtimeInterface, + Location: nextTransactionLocation(), + }, + ) + require.NoError(t, err) - var nonEmptyKeys []string - for key, data := range ledger.StoredValues { - if len(data) > 0 { - nonEmptyKeys = append(nonEmptyKeys, key) - } + var actualNonEmptyKeys []string + for key, data := range ledger.StoredValues { + if len(data) > 0 { + actualNonEmptyKeys = append(actualNonEmptyKeys, key) + } + } + + sort.Strings(actualNonEmptyKeys) + + assert.Equal(t, + expectedNonEmptyKeys, + actualNonEmptyKeys, + ) + + expectedUUID := interpreter.NewUnmeteredUInt64Value(1) + assert.Equal(t, + []resourceOwnerChange{ + { + typeID: "A.0000000000000001.Test.R", + uuid: &expectedUUID, + oldAddress: common.Address{ + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, + }, + newAddress: common.Address{ + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, + }, + }, + { + typeID: "A.0000000000000001.Test.R", + uuid: &expectedUUID, + oldAddress: common.Address{ + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, + }, + newAddress: common.Address{ + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, + }, + }, + { + typeID: "A.0000000000000001.Test.R", + uuid: &expectedUUID, + oldAddress: common.Address{ + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, + }, + newAddress: common.Address{ + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, + }, + }, + }, + resourceOwnerChanges, + ) + }) } - sort.Strings(nonEmptyKeys) + test( + false, + []string{ + // account 0x1: + // NOTE: with atree inlining, contract is inlined in contract map + // storage map (domain key + map slab) + // + contract map (domain key + map slab) + "\x00\x00\x00\x00\x00\x00\x00\x01|$\x00\x00\x00\x00\x00\x00\x00\x02", + "\x00\x00\x00\x00\x00\x00\x00\x01|$\x00\x00\x00\x00\x00\x00\x00\x04", + "\x00\x00\x00\x00\x00\x00\x00\x01|contract", + "\x00\x00\x00\x00\x00\x00\x00\x01|storage", + // account 0x2 + // NOTE: with atree inlining, resource is inlined in storage map + // storage map (domain key + map slab) + "\x00\x00\x00\x00\x00\x00\x00\x02|$\x00\x00\x00\x00\x00\x00\x00\x02", + "\x00\x00\x00\x00\x00\x00\x00\x02|storage", + }, + ) - assert.Equal(t, + test( + true, []string{ // account 0x1: // NOTE: with account storage map and atree inlining, @@ -1789,44 +1865,6 @@ func TestRuntimeResourceOwnerChange(t *testing.T) { "\x00\x00\x00\x00\x00\x00\x00\x02|$\x00\x00\x00\x00\x00\x00\x00\x02", "\x00\x00\x00\x00\x00\x00\x00\x02|stored", }, - nonEmptyKeys, - ) - - expectedUUID := interpreter.NewUnmeteredUInt64Value(1) - assert.Equal(t, - []resourceOwnerChange{ - { - typeID: "A.0000000000000001.Test.R", - uuid: &expectedUUID, - oldAddress: common.Address{ - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - }, - newAddress: common.Address{ - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, - }, - }, - { - typeID: "A.0000000000000001.Test.R", - uuid: &expectedUUID, - oldAddress: common.Address{ - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, - }, - newAddress: common.Address{ - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - }, - }, - { - typeID: "A.0000000000000001.Test.R", - uuid: &expectedUUID, - oldAddress: common.Address{ - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - }, - newAddress: common.Address{ - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, - }, - }, - }, - resourceOwnerChanges, ) } @@ -3110,7 +3148,7 @@ func TestRuntimeStorageInternalAccess(t *testing.T) { }) require.NoError(t, err) - storageMap := storage.GetStorageMap(inter, address, common.PathDomainStorage.StorageDomain(), false) + storageMap := storage.GetDomainStorageMap(inter, address, common.PathDomainStorage.StorageDomain(), false) require.NotNil(t, storageMap) // Read first @@ -6260,7 +6298,13 @@ func TestRuntimeStorageForNewAccount(t *testing.T) { // Create empty storage ledger := NewTestLedger(nil, LedgerOnWriteCounter(&writeCount)) - storage := NewStorage(ledger, nil) + storage := NewStorage( + ledger, + nil, + StorageConfig{ + StorageFormatV2Enabled: true, + }, + ) inter := NewTestInterpreterWithStorage(t, storage) @@ -6268,7 +6312,7 @@ func TestRuntimeStorageForNewAccount(t *testing.T) { // Get non-existent domain storage map const createIfNotExists = false - domainStorageMap := storage.GetStorageMap(inter, address, domain, createIfNotExists) + domainStorageMap := storage.GetDomainStorageMap(inter, address, domain, createIfNotExists) require.Nil(t, domainStorageMap) // Commit changes @@ -6309,7 +6353,13 @@ func TestRuntimeStorageForNewAccount(t *testing.T) { // Create empty storage ledger := NewTestLedger(nil, LedgerOnWriteEntries(&writeEntries)) - storage := NewStorage(ledger, nil) + storage := NewStorage( + ledger, + nil, + StorageConfig{ + StorageFormatV2Enabled: true, + }, + ) inter := NewTestInterpreterWithStorage(t, storage) @@ -6321,7 +6371,7 @@ func TestRuntimeStorageForNewAccount(t *testing.T) { for _, domain := range tc.newDomains { // Create new domain storage map const createIfNotExists = true - domainStorageMap := storage.GetStorageMap(inter, address, domain, createIfNotExists) + domainStorageMap := storage.GetDomainStorageMap(inter, address, domain, createIfNotExists) require.NotNil(t, domainStorageMap) require.Equal(t, uint64(0), domainStorageMap.Count()) @@ -6387,7 +6437,13 @@ func TestRuntimeStorageForNewAccount(t *testing.T) { t.Run("create, commit, write, commit, remove, commit", func(t *testing.T) { // Create empty storage ledger := NewTestLedger(nil, nil) - storage := NewStorage(ledger, nil) + storage := NewStorage( + ledger, + nil, + StorageConfig{ + StorageFormatV2Enabled: true, + }, + ) inter := NewTestInterpreterWithStorage(t, storage) @@ -6404,7 +6460,7 @@ func TestRuntimeStorageForNewAccount(t *testing.T) { { for _, domain := range domains { const createIfNotExists = true - domainStorageMap := storage.GetStorageMap(inter, address, domain, createIfNotExists) + domainStorageMap := storage.GetDomainStorageMap(inter, address, domain, createIfNotExists) require.NotNil(t, domainStorageMap) require.Equal(t, uint64(0), domainStorageMap.Count()) @@ -6428,7 +6484,7 @@ func TestRuntimeStorageForNewAccount(t *testing.T) { { for _, domain := range domains { const createIfNotExists = false - domainStorageMap := storage.GetStorageMap(inter, address, domain, createIfNotExists) + domainStorageMap := storage.GetDomainStorageMap(inter, address, domain, createIfNotExists) require.NotNil(t, domainStorageMap) require.Equal(t, uint64(0), domainStorageMap.Count()) @@ -6454,7 +6510,7 @@ func TestRuntimeStorageForNewAccount(t *testing.T) { { for _, domain := range domains { const createIfNotExists = false - domainStorageMap := storage.GetStorageMap(inter, address, domain, createIfNotExists) + domainStorageMap := storage.GetDomainStorageMap(inter, address, domain, createIfNotExists) require.NotNil(t, domainStorageMap) expectedDomainValues := accountValues[domain] @@ -6486,7 +6542,7 @@ func TestRuntimeStorageForNewAccount(t *testing.T) { { for _, domain := range domains { const createIfNotExists = false - domainStorageMap := storage.GetStorageMap(inter, address, domain, createIfNotExists) + domainStorageMap := storage.GetDomainStorageMap(inter, address, domain, createIfNotExists) require.NotNil(t, domainStorageMap) require.Equal(t, uint64(0), domainStorageMap.Count()) } @@ -6521,7 +6577,13 @@ func TestRuntimeStorageForMigratedAccount(t *testing.T) { domainStorageMapCount int, ) (TestLedger, accountStorageMapValues) { ledger := NewTestLedger(nil, nil) - storage := NewStorage(ledger, nil) + storage := NewStorage( + ledger, + nil, + StorageConfig{ + StorageFormatV2Enabled: true, + }, + ) inter := NewTestInterpreterWithStorage(t, storage) @@ -6555,13 +6617,19 @@ func TestRuntimeStorageForMigratedAccount(t *testing.T) { address, existingDomains, domainStorageMapCount) - storage := NewStorage(ledger, nil) + storage := NewStorage( + ledger, + nil, + StorageConfig{ + StorageFormatV2Enabled: true, + }, + ) inter := NewTestInterpreterWithStorage(t, storage) // Get non-existent domain storage map const createIfNotExists = false - domainStorageMap := storage.GetStorageMap(inter, address, nonexistentDomain, createIfNotExists) + domainStorageMap := storage.GetDomainStorageMap(inter, address, nonexistentDomain, createIfNotExists) require.Nil(t, domainStorageMap) // Commit changes @@ -6601,13 +6669,19 @@ func TestRuntimeStorageForMigratedAccount(t *testing.T) { existingDomains, domainStorageMapCount, ) - storage := NewStorage(ledger, nil) + storage := NewStorage( + ledger, + nil, + StorageConfig{ + StorageFormatV2Enabled: true, + }, + ) inter := NewTestInterpreterWithStorage(t, storage) // Read existing domain storage map for domain, domainValues := range accountValues { - domainStorageMap := storage.GetStorageMap(inter, address, domain, tc.createIfNotExists) + domainStorageMap := storage.GetDomainStorageMap(inter, address, domain, tc.createIfNotExists) require.NotNil(t, domainStorageMap) require.Equal(t, uint64(len(domainValues)), domainStorageMap.Count()) @@ -6685,7 +6759,13 @@ func TestRuntimeStorageForMigratedAccount(t *testing.T) { tc.existingDomains, tc.existingDomainStorageMapCount, ) - storage := NewStorage(ledger, nil) + storage := NewStorage( + ledger, + nil, + StorageConfig{ + StorageFormatV2Enabled: true, + }, + ) inter := NewTestInterpreterWithStorage(t, storage) @@ -6696,7 +6776,7 @@ func TestRuntimeStorageForMigratedAccount(t *testing.T) { // Create and write to domain storage map (createIfNotExists is true) for _, domain := range tc.newDomains { const createIfNotExists = true - domainStorageMap := storage.GetStorageMap(inter, address, domain, createIfNotExists) + domainStorageMap := storage.GetDomainStorageMap(inter, address, domain, createIfNotExists) require.NotNil(t, domainStorageMap) require.Equal(t, uint64(0), domainStorageMap.Count()) @@ -6770,7 +6850,13 @@ func TestRuntimeStorageForMigratedAccount(t *testing.T) { existingDomains, existingDomainStorageMapCount, ) - storage := NewStorage(ledger, nil) + storage := NewStorage( + ledger, + nil, + StorageConfig{ + StorageFormatV2Enabled: true, + }, + ) inter := NewTestInterpreterWithStorage(t, storage) @@ -6779,7 +6865,7 @@ func TestRuntimeStorageForMigratedAccount(t *testing.T) { // Write to existing domain storage map (createIfNotExists is false) for _, domain := range existingDomains { const createIfNotExists = false - domainStorageMap := storage.GetStorageMap(inter, address, domain, createIfNotExists) + domainStorageMap := storage.GetDomainStorageMap(inter, address, domain, createIfNotExists) require.NotNil(t, domainStorageMap) domainValues := accountValues[domain] @@ -6863,7 +6949,13 @@ func TestRuntimeStorageForMigratedAccount(t *testing.T) { domains, domainStorageMapCount, ) - storage := NewStorage(ledger, nil) + storage := NewStorage( + ledger, + nil, + StorageConfig{ + StorageFormatV2Enabled: true, + }, + ) inter := NewTestInterpreterWithStorage(t, storage) @@ -6873,7 +6965,7 @@ func TestRuntimeStorageForMigratedAccount(t *testing.T) { { for _, domain := range domains { const createIfNotExists = false - domainStorageMap := storage.GetStorageMap(inter, address, domain, createIfNotExists) + domainStorageMap := storage.GetDomainStorageMap(inter, address, domain, createIfNotExists) require.NotNil(t, domainStorageMap) domainValues := accountValues[domain] @@ -6904,7 +6996,7 @@ func TestRuntimeStorageForMigratedAccount(t *testing.T) { { for _, domain := range domains { const createIfNotExists = false - domainStorageMap := storage.GetStorageMap(inter, address, domain, createIfNotExists) + domainStorageMap := storage.GetDomainStorageMap(inter, address, domain, createIfNotExists) require.NotNil(t, domainStorageMap) domainValues := accountValues[domain] @@ -6935,7 +7027,7 @@ func TestRuntimeStorageForMigratedAccount(t *testing.T) { { for _, domain := range domains { const createIfNotExists = false - domainStorageMap := storage.GetStorageMap(inter, address, domain, createIfNotExists) + domainStorageMap := storage.GetDomainStorageMap(inter, address, domain, createIfNotExists) require.NotNil(t, domainStorageMap) expectedDomainValues := accountValues[domain] @@ -6967,7 +7059,7 @@ func TestRuntimeStorageForMigratedAccount(t *testing.T) { { for _, domain := range domains { const createIfNotExists = false - domainStorageMap := storage.GetStorageMap(inter, address, domain, createIfNotExists) + domainStorageMap := storage.GetDomainStorageMap(inter, address, domain, createIfNotExists) require.NotNil(t, domainStorageMap) require.Equal(t, uint64(0), domainStorageMap.Count()) } @@ -7001,7 +7093,13 @@ func TestRuntimeStorageForUnmigratedAccount(t *testing.T) { domainStorageMapCount int, ) (TestLedger, accountStorageMapValues) { ledger := NewTestLedger(nil, nil) - storage := NewStorage(ledger, nil) + storage := NewStorage( + ledger, + nil, + StorageConfig{ + StorageFormatV2Enabled: false, + }, + ) inter := NewTestInterpreter(t) @@ -7062,16 +7160,24 @@ func TestRuntimeStorageForUnmigratedAccount(t *testing.T) { address, existingDomains, domainStorageMapCount) - storage := NewStorage(ledger, nil) + storage := NewStorage( + ledger, + nil, + StorageConfig{ + StorageFormatV2Enabled: true, + }, + ) inter := NewTestInterpreterWithStorage(t, storage) // Get non-existent domain storage map const createIfNotExists = false - nonexistingDomain := common.PathDomainPublic.StorageDomain() - domainStorageMap := storage.GetStorageMap(inter, address, nonexistingDomain, createIfNotExists) + nonExistingDomain := common.PathDomainPublic.StorageDomain() + domainStorageMap := storage.GetDomainStorageMap(inter, address, nonExistingDomain, createIfNotExists) require.Nil(t, domainStorageMap) + storage.ScheduleV2MigrationForModifiedAccounts() + // Commit changes const commitContractUpdates = false err := storage.Commit(inter, commitContractUpdates) @@ -7109,17 +7215,23 @@ func TestRuntimeStorageForUnmigratedAccount(t *testing.T) { existingDomains, existingDomainStorageMapCount, ) - storage := NewStorage(ledger, nil) + storage := NewStorage( + ledger, + nil, + StorageConfig{ + StorageFormatV2Enabled: true, + }, + ) inter := NewTestInterpreterWithStorage(t, storage) // Read existing domain storage map for domain, domainValues := range accountValues { - domainStorageMap := storage.GetStorageMap(inter, address, domain, tc.createIfNotExists) + domainStorageMap := storage.GetDomainStorageMap(inter, address, domain, tc.createIfNotExists) require.NotNil(t, domainStorageMap) require.Equal(t, uint64(len(domainValues)), domainStorageMap.Count()) - // Read elements to to domain storage map + // Read elements to domain storage map for k, expectedV := range domainValues { v := domainStorageMap.ReadValue(nil, k) ev, ok := v.(interpreter.EquatableValue) @@ -7197,7 +7309,13 @@ func TestRuntimeStorageForUnmigratedAccount(t *testing.T) { tc.existingDomains, tc.existingDomainStorageMapCount, ) - storage := NewStorage(ledger, nil) + storage := NewStorage( + ledger, + nil, + StorageConfig{ + StorageFormatV2Enabled: true, + }, + ) inter := NewTestInterpreterWithStorage(t, storage) @@ -7206,7 +7324,7 @@ func TestRuntimeStorageForUnmigratedAccount(t *testing.T) { // Create and write to new domain storage map for _, domain := range tc.newDomains { const createIfNotExists = true - domainStorageMap := storage.GetStorageMap(inter, address, domain, createIfNotExists) + domainStorageMap := storage.GetDomainStorageMap(inter, address, domain, createIfNotExists) require.NotNil(t, domainStorageMap) require.Equal(t, uint64(0), domainStorageMap.Count()) @@ -7214,6 +7332,9 @@ func TestRuntimeStorageForUnmigratedAccount(t *testing.T) { accountValues[domain] = writeToDomainStorageMap(inter, domainStorageMap, tc.newDomainStorageMapCount, random) } + // TODO: + storage.ScheduleV2MigrationForModifiedAccounts() + // Commit changes const commitContractUpdates = false err := storage.Commit(inter, commitContractUpdates) @@ -7232,12 +7353,28 @@ func TestRuntimeStorageForUnmigratedAccount(t *testing.T) { require.True(t, len(writeEntries) > 1+len(tc.existingDomains)+len(tc.newDomains)) i := 0 + + // Check new domain register committed in V1 format. + for _, domain := range common.AllStorageDomains { + + if slices.Contains(tc.newDomains, domain) { + + // New domains are committed in V1 format (with domain register). + require.Equal(t, address[:], writeEntries[i].Owner) + require.Equal(t, []byte(domain.Identifier()), writeEntries[i].Key) + require.True(t, len(writeEntries[i].Value) > 0) + + i++ + } + } + + // Check modified registers in migration. for _, domain := range common.AllStorageDomains { if slices.Contains(tc.existingDomains, domain) || slices.Contains(tc.newDomains, domain) { - // Existing and new domain registers are removed. + // Existing and new domain registers are removed (migrated). // Removing new (non-existent) domain registers is no-op. require.Equal(t, address[:], writeEntries[i].Owner) require.Equal(t, []byte(domain.Identifier()), writeEntries[i].Key) @@ -7288,7 +7425,13 @@ func TestRuntimeStorageForUnmigratedAccount(t *testing.T) { domains, existingDomainStorageMapCount, ) - storage := NewStorage(ledger, nil) + storage := NewStorage( + ledger, + nil, + StorageConfig{ + StorageFormatV2Enabled: true, + }, + ) inter := NewTestInterpreterWithStorage(t, storage) @@ -7297,7 +7440,7 @@ func TestRuntimeStorageForUnmigratedAccount(t *testing.T) { // write to existing domain storage map (createIfNotExists is false) for _, domain := range domains { const createIfNotExists = false - domainStorageMap := storage.GetStorageMap(inter, address, domain, createIfNotExists) + domainStorageMap := storage.GetDomainStorageMap(inter, address, domain, createIfNotExists) require.NotNil(t, domainStorageMap) domainValues := accountValues[domain] @@ -7337,6 +7480,9 @@ func TestRuntimeStorageForUnmigratedAccount(t *testing.T) { } } + // TODO: + storage.ScheduleV2MigrationForModifiedAccounts() + // Commit changes const commitContractUpdates = false err := storage.Commit(inter, commitContractUpdates) @@ -7377,7 +7523,7 @@ func TestRuntimeStorageForUnmigratedAccount(t *testing.T) { checkAccountStorageMapData(t, ledger.StoredValues, ledger.StorageIndices, address, accountValues) }) - // This test test storage map operations (including account migration) with intermittent Commit() + // This test storage map operations (including account migration) with intermittent Commit() // - read domain storage map and commit // - write to domain storage map and commit (including account migration) // - remove all elements from domain storage map and commit @@ -7397,7 +7543,13 @@ func TestRuntimeStorageForUnmigratedAccount(t *testing.T) { domains, domainStorageMapCount, ) - storage := NewStorage(ledger, nil) + storage := NewStorage( + ledger, + nil, + StorageConfig{ + StorageFormatV2Enabled: true, + }, + ) inter := NewTestInterpreterWithStorage(t, storage) @@ -7407,7 +7559,7 @@ func TestRuntimeStorageForUnmigratedAccount(t *testing.T) { { for _, domain := range domains { const createIfNotExists = false - domainStorageMap := storage.GetStorageMap(inter, address, domain, createIfNotExists) + domainStorageMap := storage.GetDomainStorageMap(inter, address, domain, createIfNotExists) require.NotNil(t, domainStorageMap) domainValues := accountValues[domain] @@ -7436,7 +7588,7 @@ func TestRuntimeStorageForUnmigratedAccount(t *testing.T) { // update existing domain storage map (loaded from storage) for _, domain := range domains { const createIfNotExists = false - domainStorageMap := storage.GetStorageMap(inter, address, domain, createIfNotExists) + domainStorageMap := storage.GetDomainStorageMap(inter, address, domain, createIfNotExists) require.NotNil(t, domainStorageMap) domainValues := accountValues[domain] @@ -7461,6 +7613,9 @@ func TestRuntimeStorageForUnmigratedAccount(t *testing.T) { } } + // TODO: + storage.ScheduleV2MigrationForModifiedAccounts() + // Commit changes const commitContractUpdates = false err := storage.Commit(inter, commitContractUpdates) @@ -7507,7 +7662,7 @@ func TestRuntimeStorageForUnmigratedAccount(t *testing.T) { { for _, domain := range domains { const createIfNotExists = false - domainStorageMap := storage.GetStorageMap(inter, address, domain, createIfNotExists) + domainStorageMap := storage.GetDomainStorageMap(inter, address, domain, createIfNotExists) require.NotNil(t, domainStorageMap) domainValues := accountValues[domain] @@ -7556,7 +7711,7 @@ func TestRuntimeStorageForUnmigratedAccount(t *testing.T) { { for _, domain := range domains { const createIfNotExists = false - domainStorageMap := storage.GetStorageMap(inter, address, domain, createIfNotExists) + domainStorageMap := storage.GetDomainStorageMap(inter, address, domain, createIfNotExists) require.NotNil(t, domainStorageMap) domainValues := accountValues[domain] @@ -7598,7 +7753,13 @@ func TestRuntimeStorageDomainStorageMapInlinedState(t *testing.T) { // Create empty storage ledger := NewTestLedger(nil, nil) - storage := NewStorage(ledger, nil) + storage := NewStorage( + ledger, + nil, + StorageConfig{ + StorageFormatV2Enabled: true, + }, + ) inter := NewTestInterpreterWithStorage(t, storage) @@ -7616,7 +7777,7 @@ func TestRuntimeStorageDomainStorageMapInlinedState(t *testing.T) { // Create domain storage map const createIfNotExists = true - domainStorageMap := storage.GetStorageMap(inter, address, domain, createIfNotExists) + domainStorageMap := storage.GetDomainStorageMap(inter, address, domain, createIfNotExists) require.NotNil(t, domainStorageMap) require.True(t, domainStorageMap.Inlined()) @@ -7720,7 +7881,13 @@ func TestRuntimeStorageLargeDomainValues(t *testing.T) { // Create empty storage ledger := NewTestLedger(nil, nil) - storage := NewStorage(ledger, nil) + storage := NewStorage( + ledger, + nil, + StorageConfig{ + StorageFormatV2Enabled: true, + }, + ) inter := NewTestInterpreterWithStorage(t, storage) @@ -7738,7 +7905,7 @@ func TestRuntimeStorageLargeDomainValues(t *testing.T) { // Create domain storage map const createIfNotExists = true - domainStorageMap := storage.GetStorageMap(inter, address, domain, createIfNotExists) + domainStorageMap := storage.GetDomainStorageMap(inter, address, domain, createIfNotExists) require.NotNil(t, domainStorageMap) require.True(t, domainStorageMap.Inlined()) @@ -7856,7 +8023,13 @@ func TestDomainRegisterMigrationForLargeAccount(t *testing.T) { LedgerOnWriteCounter(&writeCount), accountsInfo, ) - storage := NewStorage(ledger, nil) + storage := NewStorage( + ledger, + nil, + StorageConfig{ + StorageFormatV2Enabled: true, + }, + ) inter := NewTestInterpreterWithStorage(t, storage) @@ -7865,11 +8038,14 @@ func TestDomainRegisterMigrationForLargeAccount(t *testing.T) { // Create new domain storage map const createIfNotExists = true domain := common.StorageDomainInbox - domainStorageMap := storage.GetStorageMap(inter, address, domain, createIfNotExists) + domainStorageMap := storage.GetDomainStorageMap(inter, address, domain, createIfNotExists) require.NotNil(t, domainStorageMap) accountValues[domain] = make(domainStorageMapValues) + // TODO: + storage.ScheduleV2MigrationForModifiedAccounts() + // Commit changes const commitContractUpdates = false err := storage.Commit(inter, commitContractUpdates) @@ -7921,7 +8097,7 @@ func createAndWriteAccountStorageMap( // Create domain storage map for _, domain := range domains { const createIfNotExists = true - domainStorageMap := storage.GetStorageMap(inter, address, domain, createIfNotExists) + domainStorageMap := storage.GetDomainStorageMap(inter, address, domain, createIfNotExists) require.NotNil(t, domainStorageMap) require.Equal(t, uint64(0), domainStorageMap.Count()) @@ -7964,7 +8140,8 @@ func writeToDomainStorageMap( return domainValues } -// checkAccountStorageMapData creates new storage with given storedValues, and compares account storage map values with given expectedAccountValues. +// checkAccountStorageMapData creates new storage with given storedValues, +// and compares account storage map values with given expectedAccountValues. func checkAccountStorageMapData( tb testing.TB, storedValues map[string][]byte, @@ -7974,7 +8151,13 @@ func checkAccountStorageMapData( ) { // Create storage with given storedValues and storageIndices ledger := NewTestLedgerWithData(nil, nil, storedValues, storageIndices) - storage := NewStorage(ledger, nil) + storage := NewStorage( + ledger, + nil, + StorageConfig{ + StorageFormatV2Enabled: true, + }, + ) inter := NewTestInterpreterWithStorage(tb, storage) diff --git a/runtime/transaction_executor.go b/runtime/transaction_executor.go index a8d3f30a90..f071aeb8e2 100644 --- a/runtime/transaction_executor.go +++ b/runtime/transaction_executor.go @@ -106,7 +106,13 @@ func (executor *interpreterTransactionExecutor) preprocess() (err error) { runtimeInterface := context.Interface - storage := NewStorage(runtimeInterface, runtimeInterface) + storage := NewStorage( + runtimeInterface, + runtimeInterface, + StorageConfig{ + StorageFormatV2Enabled: interpreterRuntime.defaultConfig.StorageFormatV2Enabled, + }, + ) executor.storage = storage environment := context.Environment diff --git a/stdlib/account.go b/stdlib/account.go index b3a785bf39..54bc1a1892 100644 --- a/stdlib/account.go +++ b/stdlib/account.go @@ -3242,7 +3242,7 @@ func recordStorageCapabilityController( storageMapKey := interpreter.StringStorageMapKey(identifier) - storageMap := inter.Storage().GetStorageMap( + storageMap := inter.Storage().GetDomainStorageMap( inter, address, common.StorageDomainPathCapability, @@ -3285,7 +3285,7 @@ func getPathCapabilityIDSet( storageMapKey := interpreter.StringStorageMapKey(identifier) - storageMap := inter.Storage().GetStorageMap( + storageMap := inter.Storage().GetDomainStorageMap( inter, address, common.StorageDomainPathCapability, @@ -3336,7 +3336,7 @@ func unrecordStorageCapabilityController( // Remove capability set if empty if capabilityIDSet.Count() == 0 { - storageMap := inter.Storage().GetStorageMap( + storageMap := inter.Storage().GetDomainStorageMap( inter, address, common.StorageDomainPathCapability, @@ -3405,7 +3405,7 @@ func recordAccountCapabilityController( storageMapKey := interpreter.Uint64StorageMapKey(capabilityIDValue) - storageMap := inter.Storage().GetStorageMap( + storageMap := inter.Storage().GetDomainStorageMap( inter, address, common.StorageDomainAccountCapability, @@ -3433,7 +3433,7 @@ func unrecordAccountCapabilityController( storageMapKey := interpreter.Uint64StorageMapKey(capabilityIDValue) - storageMap := inter.Storage().GetStorageMap( + storageMap := inter.Storage().GetDomainStorageMap( inter, address, common.StorageDomainAccountCapability, @@ -3453,7 +3453,7 @@ func getAccountCapabilityControllerIDsIterator( nextCapabilityID func() (uint64, bool), count uint64, ) { - storageMap := inter.Storage().GetStorageMap( + storageMap := inter.Storage().GetDomainStorageMap( inter, address, common.StorageDomainAccountCapability, diff --git a/tools/storage-explorer/main.go b/tools/storage-explorer/main.go index fe4f80504f..614143a5f7 100644 --- a/tools/storage-explorer/main.go +++ b/tools/storage-explorer/main.go @@ -184,7 +184,7 @@ func NewAccountStorageMapKeysHandler( } var keys []string - storageMap := storage.GetStorageMap(address, storageMapDomain, false) + storageMap := storage.GetDomainStorageMap(address, storageMapDomain, false) if storageMap == nil { keys = make([]string, 0) } else { @@ -225,7 +225,7 @@ func NewAccountStorageMapValueHandler( return } - storageMap := storage.GetStorageMap(address, storageMapDomain, false) + storageMap := storage.GetDomainStorageMap(address, storageMapDomain, false) if storageMap == nil { http.Error(w, "storage map does not exist", http.StatusNotFound) return