Merge pull request #557 from onflow/supun/state-migration

Support state migration for emulator state

go.mod

@@ -123,6 +123,7 @@ require (
 	github.com/mattn/go-runewidth v0.0.15 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
 	github.com/minio/sha256-simd v1.0.1 // indirect
+	github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/mmcloughlin/addchain v0.4.0 // indirect
 	github.com/mr-tron/base58 v1.2.0 // indirect
@@ -153,6 +154,7 @@ require (
 	github.com/rivo/uniseg v0.4.4 // indirect
 	github.com/rogpeppe/go-internal v1.9.0 // indirect
 	github.com/rs/cors v1.8.0 // indirect
+	github.com/schollz/progressbar/v3 v3.13.1 // indirect
 	github.com/sethvargo/go-retry v0.2.3 // indirect
 	github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible // indirect
 	github.com/slok/go-http-metrics v0.10.0 // indirect
@@ -192,6 +194,7 @@ require (
 	golang.org/x/net v0.19.0 // indirect
 	golang.org/x/sync v0.5.0 // indirect
 	golang.org/x/sys v0.15.0 // indirect
+	golang.org/x/term v0.15.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
 	golang.org/x/tools v0.16.1 // indirect

storage/migration/cadence1.go

@@ -0,0 +1,69 @@
+/*
+ * Flow Emulator
+ *
+ * Copyright 2024 Dapper Labs, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package migration
+
+import (
+	"github.com/onflow/flow-go/cmd/util/ledger/migrations"
+	"github.com/onflow/flow-go/model/flow"
+	"github.com/rs/zerolog"
+
+	"github.com/onflow/flow-emulator/storage/sqlite"
+
+	"github.com/onflow/flow-go/cmd/util/ledger/reporters"
+	"github.com/onflow/flow-go/cmd/util/ledger/util"
+)
+
+func MigrateCadence1(
+	store *sqlite.Store,
+	rwf reporters.ReportWriterFactory,
+	logger zerolog.Logger,
+) error {
+	payloads, payloadInfo, _, err := util.PayloadsAndAccountsFromEmulatorSnapshot(store.DB())
+	if err != nil {
+		return err
+	}
+
+	// TODO: >1 breaks atree storage map iteration
+	//   and requires LinkValueMigration.LinkValueMigration to be thread-safe
+	const nWorker = 1
+
+	// TODO: EVM contract is not deployed in snapshot yet, so can't update it
+	const evmContractChange = migrations.EVMContractChangeNone
+
+	// TODO:
+	var stagedContracts []migrations.StagedContract
+
+	cadence1Migrations := migrations.NewCadence1Migrations(
+		logger,
+		rwf,
+		nWorker,
+		flow.Emulator,
+		evmContractChange,
+		stagedContracts,
+	)
+
+	for _, migration := range cadence1Migrations {
+		payloads, err = migration(payloads)
+		if err != nil {
+			return err
+		}
+	}
+
+	return WritePayloadsToSnapshot(store, payloads, payloadInfo)
+}

storage/migration/cadence1_test.go

@@ -0,0 +1,77 @@
+/*
+ * Flow Emulator
+ *
+ * Copyright 2024 Dapper Labs, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package migration
+
+import (
+	"io"
+	"os"
+	"testing"
+
+	"github.com/rs/zerolog"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/onflow/flow-emulator/storage/sqlite"
+)
+
+func TestCadence1Migration(t *testing.T) {
+	const emulatorStateFile = "test-data/emulator_state_cadence_v0.42.6"
+
+	// Work on a temp copy of the state,
+	// since the migration will be updating the state.
+	tempEmulatorState, err := os.CreateTemp("test-data", "temp_emulator_state")
+	require.NoError(t, err)
+
+	tempEmulatorStatePath := tempEmulatorState.Name()
+
+	defer tempEmulatorState.Close()
+	defer os.Remove(tempEmulatorStatePath)
+
+	content, err := os.ReadFile(emulatorStateFile)
+	require.NoError(t, err)
+
+	_, err = tempEmulatorState.Write(content)
+	require.NoError(t, err)
+
+	// Migrate
+
+	store, err := sqlite.New(tempEmulatorStatePath)
+	require.NoError(t, err)
+
+	logWriter := &writer{}
+	logger := zerolog.New(logWriter).Level(zerolog.ErrorLevel)
+
+	// Then migrate the values.
+	rwf := &NOOPReportWriterFactory{}
+	err = MigrateCadence1(store, rwf, logger)
+	require.NoError(t, err)
+
+	require.Empty(t, logWriter.logs)
+}
+
+type writer struct {
+	logs []string
+}
+
+var _ io.Writer = &writer{}
+
+func (w *writer) Write(p []byte) (n int, err error) {
+	w.logs = append(w.logs, string(p))
+	return len(p), nil
+}

storage/migration/test-data/README.md

@@ -0,0 +1,3 @@
+This is the same emulator state used in https://github.com/onflow/flow-go/tree/feature/stable-cadence/cmd/util/ledger/migrations/test-data/cadence_values_migration
+
+Follow the instruction there to generate a new state, if needed.

storage/migration/utils.go

@@ -0,0 +1,117 @@
+/*
+ * Flow Emulator
+ *
+ * Copyright 2024 Dapper Labs, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package migration
+
+import (
+	"context"
+	"os"
+
+	"github.com/rs/zerolog"
+
+	"github.com/onflow/flow-emulator/storage"
+	"github.com/onflow/flow-emulator/storage/sqlite"
+
+	"github.com/onflow/flow-go/cmd/util/ledger/reporters"
+	"github.com/onflow/flow-go/cmd/util/ledger/util"
+	"github.com/onflow/flow-go/ledger"
+	"github.com/onflow/flow-go/ledger/common/convert"
+	"github.com/onflow/flow-go/model/flow"
+)
+
+func WritePayloadsToSnapshot(
+	store *sqlite.Store,
+	payloads []*ledger.Payload,
+	payloadInfoSet map[flow.RegisterID]util.PayloadMetaInfo,
+) error {
+
+	const storeName = storage.LedgerStoreName
+
+	ctx := context.TODO()
+
+	for _, payload := range payloads {
+		key, err := payload.Key()
+		if err != nil {
+			return err
+		}
+
+		registerId, err := convert.LedgerKeyToRegisterID(key)
+		if err != nil {
+			return err
+		}
+
+		registerIdBytes := []byte(registerId.String())
+
+		value := payload.Value()
+
+		payloadInfo, ok := payloadInfoSet[registerId]
+		if ok {
+			// Insert the values back with the existing height and version.
+			err = store.SetBytesWithVersionAndHeight(
+				ctx,
+				storeName,
+				registerIdBytes,
+				value,
+				payloadInfo.Version,
+				payloadInfo.Height,
+			)
+		} else {
+			// If this is a new payload, use the current block height, and default version.
+			err = store.SetBytes(
+				ctx,
+				storeName,
+				registerIdBytes,
+				value,
+			)
+		}
+
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func NewConsoleLogger() zerolog.Logger {
+	writer := zerolog.ConsoleWriter{
+		Out: os.Stdout,
+	}
+
+	return zerolog.New(writer).
+		With().
+		Timestamp().
+		Logger().
+		Level(zerolog.InfoLevel)
+}
+
+type NOOPReportWriterFactory struct{}
+
+func (*NOOPReportWriterFactory) ReportWriter(_ string) reporters.ReportWriter {
+	return &NOOPWriter{}
+}
+
+type NOOPWriter struct{}
+
+var _ reporters.ReportWriter = &NOOPWriter{}
+
+func (*NOOPWriter) Write(_ any) {
+	// NO-OP
+}
+
+func (r *NOOPWriter) Close() {}

storage/sqlite/store.go

@@ -342,3 +342,30 @@ func (s *Store) Close() error {
 
 	return nil
 }
+
+// Below are needed for the state migrations only.
+
+func (s *Store) DB() *sql.DB {
+	return s.db
+}
+
+func (s *Store) SetBytesWithVersionAndHeight(_ context.Context, store string, key []byte, value []byte, version, height uint64) error {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	_, err := s.db.Exec(
+		fmt.Sprintf(
+			"INSERT INTO %s (key, version, value, height) VALUES (?, ?, ?, ?) ON CONFLICT(key, version, height) DO UPDATE SET value=excluded.value",
+			store,
+		),
+		hex.EncodeToString(key),
+		version,
+		hex.EncodeToString(value),
+		height,
+	)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}