goal: Add export and import options for genesis files during network …

…creation (#5672)

goal network pregen [-p] [-t]: Generates genesis.json, root and part keys based on the template file [-t].
[-p] (pregendir) is where the genesis files are saved (since this is an inherited required flag),
and it must be empty or command will error.

goal network create [-r] [-t] [-p]: Imports files from pregendir to networkDir.
Importing keys was already possible, but this explicitly copies over the keys from another directory,
because network directories are wiped if there is an error during creation.

cmd/goal/network.go

@@ -28,6 +28,7 @@ import (
 
 	"github.com/algorand/go-algorand/cmd/util/datadir"
 	"github.com/algorand/go-algorand/config"
+	"github.com/algorand/go-algorand/gen"
 	"github.com/algorand/go-algorand/netdeploy"
 	"github.com/algorand/go-algorand/util"
 )
@@ -40,26 +41,46 @@ var noImportKeys bool
 var noClean bool
 var devModeOverride bool
 var startOnCreation bool
+var pregenDir string
 
 func init() {
 	networkCmd.AddCommand(networkCreateCmd)
 	networkCmd.PersistentFlags().StringVarP(&networkRootDir, "rootdir", "r", "", "Root directory for the private network directories")
-	networkCmd.MarkPersistentFlagRequired("rootdir")
 
 	networkCreateCmd.Flags().StringVarP(&networkName, "network", "n", "", "Specify the name to use for the private network")
 	networkCreateCmd.Flags().StringVarP(&networkTemplateFile, "template", "t", "", "Specify the path to the template file for the network")
 	networkCreateCmd.Flags().BoolVarP(&noImportKeys, "noimportkeys", "K", false, "Do not import root keys when creating the network (by default will import)")
 	networkCreateCmd.Flags().BoolVar(&noClean, "noclean", false, "Prevents auto-cleanup on error - for diagnosing problems")
 	networkCreateCmd.Flags().BoolVar(&devModeOverride, "devMode", false, "Forces the configuration to enable DevMode, returns an error if the template is not compatible with DevMode.")
 	networkCreateCmd.Flags().BoolVarP(&startOnCreation, "start", "s", false, "Automatically start the network after creating it.")
+	networkCreateCmd.Flags().StringVarP(&pregenDir, "pregendir", "p", "", "Specify the path to the directory with pregenerated genesis.json, root and partkeys to import into the network directory. By default, the genesis.json and keys will be generated on start. This should only be used on private networks.")
+	networkCreateCmd.MarkFlagRequired("rootdir")
 
+	networkCmd.AddCommand(networkStartCmd)
 	networkStartCmd.Flags().StringVarP(&startNode, "node", "n", "", "Specify the name of a specific node to start")
+	networkStartCmd.MarkFlagRequired("rootdir")
 
-	networkCmd.AddCommand(networkStartCmd)
 	networkCmd.AddCommand(networkRestartCmd)
+	networkRestartCmd.MarkFlagRequired("rootdir")
+
 	networkCmd.AddCommand(networkStopCmd)
+	networkStopCmd.MarkFlagRequired("rootdir")
+
 	networkCmd.AddCommand(networkStatusCmd)
+	networkStatusCmd.MarkFlagRequired("rootdir")
+
 	networkCmd.AddCommand(networkDeleteCmd)
+	networkDeleteCmd.MarkFlagRequired("rootdir")
+
+	networkCmd.AddCommand(networkPregenCmd)
+	networkPregenCmd.Flags().StringVarP(&networkTemplateFile, "template", "t", "", "Specify the path to the template file for the network")
+	networkPregenCmd.Flags().StringVarP(&pregenDir, "pregendir", "p", "", "Specify the path to the directory to export genesis.json, root and partkey files. This should only be used on private networks.")
+	networkPregenCmd.MarkFlagRequired("pregendir")
+	// Hide rootdir flag as it is unused and will error if used with this command.
+	networkPregenCmd.SetHelpFunc(func(command *cobra.Command, strings []string) {
+		_ = command.Flags().MarkHidden("rootdir")
+		command.Parent().HelpFunc()(command, strings)
+	})
 }
 
 var networkCmd = &cobra.Command{
@@ -112,6 +133,18 @@ var networkCreateCmd = &cobra.Command{
 			reportErrorf(infoNetworkAlreadyExists, networkRootDir)
 		}
 
+		// If pregendir is specified, copy files over
+		if pregenDir != "" {
+			pregenDir, err = filepath.Abs(pregenDir)
+			if err != nil {
+				panic(err)
+			}
+			err = util.CopyFolder(pregenDir, networkRootDir)
+			if err != nil {
+				panic(err)
+			}
+		}
+
 		binDir, err := util.ExeDir()
 		if err != nil {
 			panic(err)
@@ -246,3 +279,64 @@ var networkDeleteCmd = &cobra.Command{
 		reportInfof(infoNetworkDeleted, networkRootDir)
 	},
 }
+
+var networkPregenCmd = &cobra.Command{
+	Use:   "pregen",
+	Short: "Pregenerates the genesis.json, root and participation keys for a wallet",
+	Args:  validateNoPosArgsFn,
+	Run: func(cmd *cobra.Command, _ []string) {
+		var err error
+		if networkRootDir != "" {
+			reportErrorf("This command does not take a network directory as an argument. Use --pregendir flag instead.")
+		}
+
+		pregenDir, err = filepath.Abs(pregenDir)
+		if err != nil {
+			panic(err)
+		}
+
+		var templateReader io.Reader
+
+		if networkTemplateFile == "" {
+			templateReader = strings.NewReader(defaultNetworkTemplate)
+		} else {
+			networkTemplateFile, err = filepath.Abs(networkTemplateFile)
+			if err != nil {
+				panic(err)
+			}
+			file, osErr := os.Open(networkTemplateFile)
+			if osErr != nil {
+				reportErrorf(errorCreateNetwork, osErr)
+			}
+
+			defer file.Close()
+			templateReader = file
+		}
+
+		// Make sure target directory does not exist or is empty
+		if util.FileExists(pregenDir) && !util.IsEmpty(pregenDir) {
+			reportErrorf(infoNetworkAlreadyExists, pregenDir)
+		}
+
+		var template netdeploy.NetworkTemplate
+		err = netdeploy.LoadTemplateFromReader(templateReader, &template)
+		if err != nil {
+			reportErrorf("Error in loading template: %v\n", err)
+		}
+
+		dataDir := datadir.MaybeSingleDataDir()
+		var consensus config.ConsensusProtocols
+		if dataDir != "" {
+			// try to load the consensus from there. If there is none, we can just use the built in one.
+			consensus, _ = config.PreloadConfigurableConsensusProtocols(dataDir)
+		}
+		if err = template.Validate(); err != nil {
+			reportErrorf("Error in template validation: %v\n", err)
+		}
+
+		err = gen.GenerateGenesisFiles(template.Genesis, config.Consensus.Merge(consensus), pregenDir, os.Stdout)
+		if err != nil {
+			reportErrorf("Cannot write genesis files: %s", err)
+		}
+	},
+}

netdeploy/network.go

@@ -70,7 +70,7 @@ func CreateNetworkFromTemplate(name, rootDir string, templateReader io.Reader, b
 	var err error
 	template := defaultNetworkTemplate
 
-	err = loadTemplateFromReader(templateReader, &template)
+	err = LoadTemplateFromReader(templateReader, &template)
 
 	if err == nil {
 		if overrideDevMode {
@@ -100,7 +100,7 @@ func CreateNetworkFromTemplate(name, rootDir string, templateReader io.Reader, b
 		return n, err
 	}
 	template.Consensus = consensus
-	err = template.generateGenesisAndWallets(rootDir, n.cfg.Name, binDir)
+	err = template.generateGenesisAndWallets(rootDir, n.cfg.Name)
 	if err != nil {
 		return n, err
 	}

netdeploy/networkTemplate.go

@@ -48,7 +48,7 @@ var defaultNetworkTemplate = NetworkTemplate{
 	Genesis: gen.DefaultGenesis,
 }
 
-func (t NetworkTemplate) generateGenesisAndWallets(targetFolder, networkName, binDir string) error {
+func (t NetworkTemplate) generateGenesisAndWallets(targetFolder, networkName string) error {
 	genesisData := t.Genesis
 	genesisData.NetworkName = networkName
 	mergedConsensus := config.Consensus.Merge(t.Consensus)
@@ -180,11 +180,12 @@ func loadTemplate(templateFile string) (NetworkTemplate, error) {
 	}
 	defer f.Close()
 
-	err = loadTemplateFromReader(f, &template)
+	err = LoadTemplateFromReader(f, &template)
 	return template, err
 }
 
-func loadTemplateFromReader(reader io.Reader, template *NetworkTemplate) error {
+// LoadTemplateFromReader loads and decodes a network template
+func LoadTemplateFromReader(reader io.Reader, template *NetworkTemplate) error {
 
 	if runtime.GOARCH == "arm" || runtime.GOARCH == "arm64" {
 		// for arm machines, use smaller key dilution

netdeploy/networkTemplates_test.go

@@ -69,9 +69,8 @@ func TestGenerateGenesis(t *testing.T) {
 
 	targetFolder := t.TempDir()
 	networkName := "testGenGen"
-	binDir := os.ExpandEnv("${GOPATH}/bin")
 
-	err := template.generateGenesisAndWallets(targetFolder, networkName, binDir)
+	err := template.generateGenesisAndWallets(targetFolder, networkName)
 	a.NoError(err)
 	_, err = os.Stat(filepath.Join(targetFolder, config.GenesisJSONFile))
 	fileExists := err == nil

test/e2e-go/features/privatenet/privatenet_test.go

@@ -0,0 +1,62 @@
+// Copyright (C) 2019-2023 Algorand, Inc.
+// This file is part of go-algorand
+//
+// go-algorand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// go-algorand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with go-algorand.  If not, see <https://www.gnu.org/licenses/>.
+
+// Check that private networks are started as designed.
+package privatenet
+
+import (
+	"testing"
+
+	"github.com/algorand/go-algorand/test/framework/fixtures"
+	"github.com/algorand/go-algorand/test/partitiontest"
+	"github.com/stretchr/testify/require"
+)
+
+// TestPrivateNetworkImportKeys tests that part keys can be exported and
+// imported when starting a private network.
+func TestPrivateNetworkImportKeys(t *testing.T) {
+	partitiontest.PartitionTest(t)
+
+	// This test takes 5~10 seconds.
+	if testing.Short() {
+		t.Skip()
+	}
+
+	// First test that keys can be exported by using `goal network pregen ...`
+	// Don't start up network, just create genesis files.
+	var goalFixture fixtures.GoalFixture
+	tmpGenDir := t.TempDir()
+	tmpNetDir := t.TempDir()
+	defaultTemplate := "" // Use the default template by omitting the filepath.
+
+	_, err := goalFixture.NetworkPregen(defaultTemplate, tmpGenDir)
+	require.NoError(t, err)
+
+	// Check that if there is an existing directory with same name, test fails.
+	errStr, err := goalFixture.NetworkPregen(defaultTemplate, tmpGenDir)
+	require.Error(t, err)
+	require.Contains(t, errStr, "already exists and is not empty")
+
+	// Then try importing files from same template.
+	err = goalFixture.NetworkCreate(tmpNetDir, "", defaultTemplate, tmpGenDir)
+	require.NoError(t, err)
+
+	err = goalFixture.NetworkStart(tmpNetDir)
+	require.NoError(t, err)
+
+	err = goalFixture.NetworkStop(tmpNetDir)
+	require.NoError(t, err)
+}

test/framework/fixtures/goalFixture.go

@@ -58,19 +58,27 @@ const (
 	nodeCmd  = "node"
 	startCmd = "start"
 	stopCmd  = "stop"
+
+	networkCmd = "network"
+	pregenCmd  = "pregen"
+	createCmd  = "create"
 )
 
-func (f *GoalFixture) executeCommand(args ...string) (retStdout string, retStderr string, err error) {
+func (f *GoalFixture) executeRawCommand(args ...string) (retStdout string, retStderr string, err error) {
+	// Executes a command without a specified data directory
 	cmd := filepath.Join(f.binDir, goalCmd)
-	// We always execute goal against the PrimaryDataDir() instance
-	args = append(args, "-d", f.PrimaryDataDir())
 	retStdout, retStderr, err = util.ExecAndCaptureOutput(cmd, args...)
 	retStdout = strings.TrimRight(retStdout, "\n")
 	retStderr = strings.TrimRight(retStderr, "\n")
-	//fmt.Printf("command: %v %v\nret: %v\n", cmd, args, ret)
 	return
 }
 
+func (f *GoalFixture) executeCommand(args ...string) (retStdout string, retStderr string, err error) {
+	// We always execute goal against the PrimaryDataDir() instance
+	args = append(args, "-d", f.PrimaryDataDir())
+	return f.executeRawCommand(args...)
+}
+
 // combine the error and the output so that we could return it as a single error object.
 func combineExecuteError(retStdout string, retStderr string, err error) error {
 	if err == nil {
@@ -227,3 +235,63 @@ func (f *GoalFixture) AccountImportRootKey(wallet string, createDefaultUnencrypt
 	_, _, err = f.executeCommand(args...)
 	return
 }
+
+// NetworkPregen exposes the `goal network pregen` command
+func (f *GoalFixture) NetworkPregen(template, pregendir string) (stdErr string, err error) {
+	args := []string{
+		networkCmd,
+		pregenCmd,
+		"-p",
+		pregendir,
+	}
+	if template != "" {
+		args = append(args, "-t", template)
+	}
+	_, stdErr, err = f.executeRawCommand(args...)
+	return
+}
+
+// NetworkCreate exposes the `goal network create` command
+func (f *GoalFixture) NetworkCreate(networkdir, networkName, template, pregendir string) (err error) {
+	args := []string{
+		networkCmd,
+		createCmd,
+		"-r",
+		networkdir,
+	}
+	if networkName != "" {
+		args = append(args, "-n", networkName)
+	}
+	if template != "" {
+		args = append(args, "-t", template)
+	}
+	if pregendir != "" {
+		args = append(args, "-p", pregendir)
+	}
+	_, _, err = f.executeRawCommand(args...)
+	return
+}
+
+// NetworkStart exposes the `goal network start` command
+func (f *GoalFixture) NetworkStart(networkdir string) (err error) {
+	args := []string{
+		networkCmd,
+		startCmd,
+		"-r",
+		networkdir,
+	}
+	_, _, err = f.executeRawCommand(args...)
+	return
+}
+
+// NetworkStop exposes the `goal network stop` command
+func (f *GoalFixture) NetworkStop(networkdir string) (err error) {
+	args := []string{
+		networkCmd,
+		stopCmd,
+		"-r",
+		networkdir,
+	}
+	_, _, err = f.executeRawCommand(args...)
+	return
+}

test/scripts/test_private_network.sh

@@ -1,8 +1,10 @@
 #!/usr/bin/env bash
+
+set -euf -o pipefail
+
 echo "######################################################################"
 echo "  test_private_network"
 echo "######################################################################"
-set -e
 
 # Suppress telemetry reporting for tests
 export ALGOTEST=1