Merge pull request #481 from onflow/fxamacker/export-persistentslabst…

…orage-funcs-for-testing

Export PersistentSlabStorage funcs in export_test.go

array_benchmark_test.go

@@ -168,22 +168,25 @@ func benchmarkArray(b *testing.B, initialArraySize, numberOfElements int) {
 	require.NoError(b, storage.Commit())
 	totalLookupTime = time.Since(start)
 
+	baseStorage := GetBaseStorage(storage)
+
 	// random lookup
-	storage.baseStorage.ResetReporter()
+	baseStorage.ResetReporter()
 	storage.DropCache()
 	array, err = NewArrayWithRootID(storage, arrayID)
 	require.NoError(b, err)
 
 	ind := r.Intn(int(array.Count()))
 	_, err = array.Get(uint64(ind))
 	require.NoError(b, err)
-	storageOverheadRatio := float64(storage.baseStorage.Size()) / float64(totalRawDataSize)
-	b.ReportMetric(float64(storage.baseStorage.SegmentsTouched()), "segments_touched")
-	b.ReportMetric(float64(storage.baseStorage.SegmentCounts()), "segments_total")
+
+	storageOverheadRatio := float64(baseStorage.Size()) / float64(totalRawDataSize)
+	b.ReportMetric(float64(baseStorage.SegmentsTouched()), "segments_touched")
+	b.ReportMetric(float64(baseStorage.SegmentCounts()), "segments_total")
 	b.ReportMetric(float64(totalRawDataSize), "storage_raw_data_size")
-	b.ReportMetric(float64(storage.baseStorage.Size()), "storage_stored_data_size")
+	b.ReportMetric(float64(baseStorage.Size()), "storage_stored_data_size")
 	b.ReportMetric(storageOverheadRatio, "storage_overhead_ratio")
-	b.ReportMetric(float64(storage.baseStorage.BytesRetrieved()), "storage_bytes_loaded_for_lookup")
+	b.ReportMetric(float64(baseStorage.BytesRetrieved()), "storage_bytes_loaded_for_lookup")
 	// b.ReportMetric(float64(array.Count()), "number_of_elements")
 	b.ReportMetric(float64(int(totalAppendTime)), "append_100_time_(ns)")
 	b.ReportMetric(float64(int(totalRemoveTime)), "remove_100_time_(ns)")
@@ -247,10 +250,12 @@ func benchmarkLongTermImpactOnMemory(b *testing.B, initialArraySize, numberOfOps
 	}
 	require.NoError(b, storage.Commit())
 
-	storageOverheadRatio := float64(storage.baseStorage.Size()) / float64(totalRawDataSize)
-	b.ReportMetric(float64(storage.baseStorage.SegmentsTouched()), "segments_touched")
-	b.ReportMetric(float64(storage.baseStorage.SegmentCounts()), "segments_total")
+	baseStorage := GetBaseStorage(storage)
+
+	storageOverheadRatio := float64(baseStorage.Size()) / float64(totalRawDataSize)
+	b.ReportMetric(float64(baseStorage.SegmentsTouched()), "segments_touched")
+	b.ReportMetric(float64(baseStorage.SegmentCounts()), "segments_total")
 	b.ReportMetric(float64(totalRawDataSize), "storage_raw_data_size")
-	b.ReportMetric(float64(storage.baseStorage.Size()), "storage_stored_data_size")
+	b.ReportMetric(float64(baseStorage.Size()), "storage_stored_data_size")
 	b.ReportMetric(storageOverheadRatio, "storage_overhead_ratio")
 }

array_test.go

@@ -119,8 +119,8 @@ func _testArray(
 	// Verify slab serializations
 	err = VerifyArraySerialization(
 		array,
-		storage.cborDecMode,
-		storage.cborEncMode,
+		GetCBORDecMode(storage),
+		GetCBOREncMode(storage),
 		storage.DecodeStorable,
 		storage.DecodeTypeInfo,
 		func(a, b Storable) bool {
@@ -144,18 +144,19 @@ func _testArray(
 	require.Equal(t, array.SlabID(), rootIDs[0])
 
 	// Encode all non-nil slab
+	deltas := GetDeltas(storage)
 	encodedSlabs := make(map[SlabID][]byte)
-	for id, slab := range storage.deltas {
+	for id, slab := range deltas {
 		if slab != nil {
-			b, err := EncodeSlab(slab, storage.cborEncMode)
+			b, err := EncodeSlab(slab, GetCBOREncMode(storage))
 			require.NoError(t, err)
 			encodedSlabs[id] = b
 		}
 	}
 
 	// Test decoded array from new storage to force slab decoding
 	decodedArray, err := NewArrayWithRootID(
-		newTestPersistentStorageWithBaseStorageAndDeltas(t, storage.baseStorage, encodedSlabs),
+		newTestPersistentStorageWithBaseStorageAndDeltas(t, GetBaseStorage(storage), encodedSlabs),
 		array.SlabID())
 	require.NoError(t, err)
 
@@ -4351,7 +4352,7 @@ func TestArrayEncodeDecodeRandomValues(t *testing.T) {
 	testArray(t, storage, typeInfo, address, array, values, false)
 
 	// Decode data to new storage
-	storage2 := newTestPersistentStorageWithBaseStorage(t, storage.baseStorage)
+	storage2 := newTestPersistentStorageWithBaseStorage(t, GetBaseStorage(storage))
 
 	// Test new array from storage2
 	array2, err := NewArrayWithRootID(storage2, array.SlabID())
@@ -5195,7 +5196,7 @@ func TestArrayLoadedValueIterator(t *testing.T) {
 		require.NoError(t, err)
 
 		// parent array: 1 root data slab
-		require.Equal(t, 1, len(storage.deltas))
+		require.Equal(t, 1, GetDeltasCount(storage))
 		require.Equal(t, 0, getArrayMetaDataSlabCount(storage))
 
 		testArrayLoadedElements(t, array, nil)
@@ -5209,7 +5210,7 @@ func TestArrayLoadedValueIterator(t *testing.T) {
 			array, values := createArrayWithSimpleValues(t, storage, address, typeInfo, arraySize, useWrapperValue)
 
 			// parent array: 1 root data slab
-			require.Equal(t, 1, len(storage.deltas))
+			require.Equal(t, 1, GetDeltasCount(storage))
 			require.Equal(t, 0, getArrayMetaDataSlabCount(storage))
 
 			testArrayLoadedElements(t, array, values)
@@ -5225,7 +5226,7 @@ func TestArrayLoadedValueIterator(t *testing.T) {
 
 			// parent array: 1 root data slab
 			// nested composite elements: 1 root data slab for each
-			require.Equal(t, 1+arraySize, len(storage.deltas))
+			require.Equal(t, 1+arraySize, GetDeltasCount(storage))
 			require.Equal(t, 0, getArrayMetaDataSlabCount(storage))
 
 			testArrayLoadedElements(t, array, values)
@@ -5241,7 +5242,7 @@ func TestArrayLoadedValueIterator(t *testing.T) {
 
 			// parent array: 1 root data slab
 			// nested composite elements: 1 root data slab for each
-			require.Equal(t, 1+arraySize, len(storage.deltas))
+			require.Equal(t, 1+arraySize, GetDeltasCount(storage))
 			require.Equal(t, 0, getArrayMetaDataSlabCount(storage))
 
 			testArrayLoadedElements(t, array, values)
@@ -5268,7 +5269,7 @@ func TestArrayLoadedValueIterator(t *testing.T) {
 
 			// parent array: 1 root data slab
 			// nested composite elements: 1 root data slab for each
-			require.Equal(t, 1+arraySize, len(storage.deltas))
+			require.Equal(t, 1+arraySize, GetDeltasCount(storage))
 			require.Equal(t, 0, getArrayMetaDataSlabCount(storage))
 
 			testArrayLoadedElements(t, array, values)
@@ -5295,7 +5296,7 @@ func TestArrayLoadedValueIterator(t *testing.T) {
 
 			// parent array: 1 root data slab
 			// nested composite elements: 1 root data slab for each
-			require.Equal(t, 1+arraySize, len(storage.deltas))
+			require.Equal(t, 1+arraySize, GetDeltasCount(storage))
 			require.Equal(t, 0, getArrayMetaDataSlabCount(storage))
 
 			testArrayLoadedElements(t, array, values)
@@ -5324,7 +5325,7 @@ func TestArrayLoadedValueIterator(t *testing.T) {
 
 			// parent array: 1 root data slab
 			// nested composite elements: 1 root data slab for each
-			require.Equal(t, 1+arraySize, len(storage.deltas))
+			require.Equal(t, 1+arraySize, GetDeltasCount(storage))
 			require.Equal(t, 0, getArrayMetaDataSlabCount(storage))
 
 			testArrayLoadedElements(t, array, values)
@@ -5362,7 +5363,7 @@ func TestArrayLoadedValueIterator(t *testing.T) {
 
 				// parent array: 1 root data slab
 				// nested composite element: 1 root data slab
-				require.Equal(t, 2, len(storage.deltas))
+				require.Equal(t, 2, GetDeltasCount(storage))
 				require.Equal(t, 0, getArrayMetaDataSlabCount(storage))
 
 				testArrayLoadedElements(t, array, values)
@@ -5387,7 +5388,7 @@ func TestArrayLoadedValueIterator(t *testing.T) {
 			array, values := createArrayWithSimpleValues(t, storage, address, typeInfo, arraySize, useWrapperValue)
 
 			// parent array: 1 root metadata slab, 2 data slabs
-			require.Equal(t, 3, len(storage.deltas))
+			require.Equal(t, 3, GetDeltasCount(storage))
 			require.Equal(t, 1, getArrayMetaDataSlabCount(storage))
 
 			testArrayLoadedElements(t, array, values)
@@ -5403,7 +5404,7 @@ func TestArrayLoadedValueIterator(t *testing.T) {
 
 			// parent array: 1 root metadata slab, 2 data slabs
 			// nested composite value element: 1 root data slab for each
-			require.Equal(t, 3+arraySize, len(storage.deltas))
+			require.Equal(t, 3+arraySize, GetDeltasCount(storage))
 			require.Equal(t, 1, getArrayMetaDataSlabCount(storage))
 
 			testArrayLoadedElements(t, array, values)
@@ -5419,7 +5420,7 @@ func TestArrayLoadedValueIterator(t *testing.T) {
 
 			// parent array: 1 root metadata slab, 2 data slabs
 			// nested composite value element: 1 root data slab for each
-			require.Equal(t, 3+arraySize, len(storage.deltas))
+			require.Equal(t, 3+arraySize, GetDeltasCount(storage))
 			require.Equal(t, 1, getArrayMetaDataSlabCount(storage))
 
 			testArrayLoadedElements(t, array, values)
@@ -5446,7 +5447,7 @@ func TestArrayLoadedValueIterator(t *testing.T) {
 
 			// parent array: 1 root metadata slab, 2 data slabs
 			// nested composite value element: 1 root data slab for each
-			require.Equal(t, 3+arraySize, len(storage.deltas))
+			require.Equal(t, 3+arraySize, GetDeltasCount(storage))
 			require.Equal(t, 1, getArrayMetaDataSlabCount(storage))
 
 			testArrayLoadedElements(t, array, values)
@@ -5473,7 +5474,7 @@ func TestArrayLoadedValueIterator(t *testing.T) {
 
 			// parent array: 1 root metadata slab, 2 data slabs
 			// nested composite value element: 1 root data slab for each
-			require.Equal(t, 3+arraySize, len(storage.deltas))
+			require.Equal(t, 3+arraySize, GetDeltasCount(storage))
 			require.Equal(t, 1, getArrayMetaDataSlabCount(storage))
 
 			testArrayLoadedElements(t, array, values)
@@ -5506,7 +5507,7 @@ func TestArrayLoadedValueIterator(t *testing.T) {
 
 				// parent array: 1 root metadata slab, 2 data slabs
 				// nested composite value element: 1 root data slab for each
-				require.Equal(t, 3+1, len(storage.deltas))
+				require.Equal(t, 3+1, GetDeltasCount(storage))
 				require.Equal(t, 1, getArrayMetaDataSlabCount(storage))
 
 				testArrayLoadedElements(t, array, values)
@@ -5531,7 +5532,7 @@ func TestArrayLoadedValueIterator(t *testing.T) {
 			array, values := createArrayWithSimpleValues(t, storage, address, typeInfo, arraySize, useWrapperValue)
 
 			// parent array (2 levels): 1 root metadata slab, 3 data slabs
-			require.Equal(t, 4, len(storage.deltas))
+			require.Equal(t, 4, GetDeltasCount(storage))
 			require.Equal(t, 1, getArrayMetaDataSlabCount(storage))
 
 			testArrayLoadedElements(t, array, values)
@@ -5562,7 +5563,7 @@ func TestArrayLoadedValueIterator(t *testing.T) {
 			array, values := createArrayWithSimpleValues(t, storage, address, typeInfo, arraySize, useWrapperValue)
 
 			// parent array (2 levels): 1 root metadata slab, 3 data slabs
-			require.Equal(t, 4, len(storage.deltas))
+			require.Equal(t, 4, GetDeltasCount(storage))
 			require.Equal(t, 1, getArrayMetaDataSlabCount(storage))
 
 			testArrayLoadedElements(t, array, values)
@@ -5593,7 +5594,7 @@ func TestArrayLoadedValueIterator(t *testing.T) {
 			array, values := createArrayWithSimpleValues(t, storage, address, typeInfo, arraySize, useWrapperValue)
 
 			// parent array (2 levels): 1 root metadata slab, 3 data slabs
-			require.Equal(t, 4, len(storage.deltas))
+			require.Equal(t, 4, GetDeltasCount(storage))
 			require.Equal(t, 1, getArrayMetaDataSlabCount(storage))
 
 			testArrayLoadedElements(t, array, values)
@@ -5681,7 +5682,7 @@ func TestArrayLoadedValueIterator(t *testing.T) {
 
 			// parent array (3 levels): 1 root metadata slab, n non-root metadata slabs, n data slabs
 			// nested composite elements: 1 root data slab for each
-			require.True(t, len(storage.deltas) > 1+arraySize)
+			require.True(t, GetDeltasCount(storage) > 1+arraySize)
 			require.True(t, getArrayMetaDataSlabCount(storage) > 1)
 
 			testArrayLoadedElements(t, array, values)
@@ -5718,7 +5719,7 @@ func TestArrayLoadedValueIterator(t *testing.T) {
 
 			// parent array (3 levels): 1 root metadata slab, n non-root metadata slabs, n data slabs
 			// nested composite elements: 1 root data slab for each
-			require.True(t, len(storage.deltas) > 1+arraySize)
+			require.True(t, GetDeltasCount(storage) > 1+arraySize)
 			require.True(t, getArrayMetaDataSlabCount(storage) > 1)
 
 			testArrayLoadedElements(t, array, values)
@@ -5735,7 +5736,7 @@ func TestArrayLoadedValueIterator(t *testing.T) {
 			count := 0
 			var dataSlabInfos []*slabInfo
 			for _, mheader := range rootMetaDataSlab.childrenHeaders {
-				nonrootMetaDataSlab, ok := storage.deltas[mheader.slabID].(*ArrayMetaDataSlab)
+				nonrootMetaDataSlab, ok := GetDeltas(storage)[mheader.slabID].(*ArrayMetaDataSlab)
 				require.True(t, ok)
 
 				for _, h := range nonrootMetaDataSlab.childrenHeaders {
@@ -5784,7 +5785,7 @@ func TestArrayLoadedValueIterator(t *testing.T) {
 
 			// parent array (3 levels): 1 root metadata slab, n non-root metadata slabs, n data slabs
 			// nested composite elements: 1 root data slab for each
-			require.True(t, len(storage.deltas) > 1+arraySize)
+			require.True(t, GetDeltasCount(storage) > 1+arraySize)
 			require.True(t, getArrayMetaDataSlabCount(storage) > 1)
 
 			testArrayLoadedElements(t, array, values)
@@ -5810,7 +5811,7 @@ func TestArrayLoadedValueIterator(t *testing.T) {
 				}
 				metadataSlabCount += int(mheader.count)
 
-				nonrootMetadataSlab, ok := storage.deltas[mheader.slabID].(*ArrayMetaDataSlab)
+				nonrootMetadataSlab, ok := GetDeltas(storage)[mheader.slabID].(*ArrayMetaDataSlab)
 				require.True(t, ok)
 
 				children := make([]*slabInfo, len(nonrootMetadataSlab.childrenHeaders))
@@ -6090,7 +6091,8 @@ func testArrayLoadedElements(t *testing.T, array *Array, expectedValues []Value)
 
 func getArrayMetaDataSlabCount(storage *PersistentSlabStorage) int {
 	var counter int
-	for _, slab := range storage.deltas {
+	deltas := GetDeltas(storage)
+	for _, slab := range deltas {
 		if _, ok := slab.(*ArrayMetaDataSlab); ok {
 			counter++
 		}
@@ -8118,7 +8120,8 @@ func createArrayWithEmpty2LevelChildArray(
 
 func getStoredDeltas(storage *PersistentSlabStorage) int {
 	count := 0
-	for _, slab := range storage.deltas {
+	deltas := GetDeltas(storage)
+	for _, slab := range deltas {
 		if slab != nil {
 			count++
 		}
@@ -8791,7 +8794,7 @@ func TestArraySetType(t *testing.T) {
 		err = storage.FastCommit(runtime.NumCPU())
 		require.NoError(t, err)
 
-		testExistingArraySetType(t, array.SlabID(), storage.baseStorage, newTypeInfo, array.Count())
+		testExistingArraySetType(t, array.SlabID(), GetBaseStorage(storage), newTypeInfo, array.Count())
 	})
 
 	t.Run("data slab root", func(t *testing.T) {
@@ -8819,7 +8822,7 @@ func TestArraySetType(t *testing.T) {
 		err = storage.FastCommit(runtime.NumCPU())
 		require.NoError(t, err)
 
-		testExistingArraySetType(t, array.SlabID(), storage.baseStorage, newTypeInfo, array.Count())
+		testExistingArraySetType(t, array.SlabID(), GetBaseStorage(storage), newTypeInfo, array.Count())
 	})
 
 	t.Run("metadata slab root", func(t *testing.T) {
@@ -8847,7 +8850,7 @@ func TestArraySetType(t *testing.T) {
 		err = storage.FastCommit(runtime.NumCPU())
 		require.NoError(t, err)
 
-		testExistingArraySetType(t, array.SlabID(), storage.baseStorage, newTypeInfo, array.Count())
+		testExistingArraySetType(t, array.SlabID(), GetBaseStorage(storage), newTypeInfo, array.Count())
 	})
 
 	t.Run("inlined in parent container root data slab", func(t *testing.T) {
@@ -8880,7 +8883,7 @@ func TestArraySetType(t *testing.T) {
 		err = storage.FastCommit(runtime.NumCPU())
 		require.NoError(t, err)
 
-		testExistingInlinedArraySetType(t, parentArray.SlabID(), 0, storage.baseStorage, newTypeInfo, childArray.Count())
+		testExistingInlinedArraySetType(t, parentArray.SlabID(), 0, GetBaseStorage(storage), newTypeInfo, childArray.Count())
 	})
 
 	t.Run("inlined in parent container non-root data slab", func(t *testing.T) {
@@ -8920,7 +8923,7 @@ func TestArraySetType(t *testing.T) {
 		err = storage.FastCommit(runtime.NumCPU())
 		require.NoError(t, err)
 
-		testExistingInlinedArraySetType(t, parentArray.SlabID(), arraySize-1, storage.baseStorage, newTypeInfo, childArray.Count())
+		testExistingInlinedArraySetType(t, parentArray.SlabID(), arraySize-1, GetBaseStorage(storage), newTypeInfo, childArray.Count())
 	})
 }
 
@@ -8953,7 +8956,7 @@ func testExistingArraySetType(
 	require.NoError(t, err)
 
 	// Create storage from existing data
-	storage2 := newTestPersistentStorageWithBaseStorage(t, storage.baseStorage)
+	storage2 := newTestPersistentStorageWithBaseStorage(t, GetBaseStorage(storage))
 
 	// Load existing array again from storage
 	array2, err := NewArrayWithRootID(storage2, id)
@@ -8999,7 +9002,7 @@ func testExistingInlinedArraySetType(
 	require.NoError(t, err)
 
 	// Create storage from existing data
-	storage2 := newTestPersistentStorageWithBaseStorage(t, storage.baseStorage)
+	storage2 := newTestPersistentStorageWithBaseStorage(t, GetBaseStorage(storage))
 
 	// Load existing array again from storage
 	parentArray2, err := NewArrayWithRootID(storage2, parentID)