Merge pull request #8642 from filecoin-project/feat/nv16-bundles

[nv16] dynamic loading of builtin actor bundles

.circleci/config.yml

@@ -48,7 +48,6 @@ commands:
             - run: sudo apt-get install ocl-icd-opencl-dev libhwloc-dev
       - run: git submodule sync
       - run: git submodule update --init
-      - fetch_builtin_actors
   download-params:
     steps:
       - restore_cache:
@@ -84,12 +83,6 @@ commands:
           name: fetch all tags
           command: |
             git fetch --all
-  fetch_builtin_actors:
-    steps:
-      - run:
-          name: fetch builtin actor bundles
-          command: |
-            build/builtin-actors/fetch-bundles.sh
   packer_build:
     description: "Run a packer build"
     parameters:

.circleci/template.yml

@@ -48,7 +48,6 @@ commands:
             - run: sudo apt-get install ocl-icd-opencl-dev libhwloc-dev
       - run: git submodule sync
       - run: git submodule update --init
-      - fetch_builtin_actors
   download-params:
     steps:
       - restore_cache:
@@ -84,12 +83,6 @@ commands:
           name: fetch all tags
           command: |
             git fetch --all
-  fetch_builtin_actors:
-    steps:
-      - run:
-          name: fetch builtin actor bundles
-          command: |
-            build/builtin-actors/fetch-bundles.sh
   packer_build:
     description: "Run a packer build"
     parameters:

Makefile

@@ -57,14 +57,6 @@ build/.update-modules:
 
 # end git modules
 
-# builtin actor bundles
-builtin-actor-bundles:
-	./build/builtin-actors/fetch-bundles.sh
-
-BUILD_DEPS+=builtin-actor-bundles
-
-.PHONY: builtin-actor-bundles
-
 ## MAIN BINARIES
 
 CLEAN+=build/.update-modules
@@ -79,15 +71,19 @@ debug: GOFLAGS+=-tags=debug
 debug: build-devnets
 
 2k: GOFLAGS+=-tags=2k
+2k: GOFLAGS+=-ldflags=-X=github.com/filecoin-project/lotus/build.NetworkBundle=devnet
 2k: build-devnets
 
 calibnet: GOFLAGS+=-tags=calibnet
+calibnet: GOFLAGS+=-ldflags=-X=github.com/filecoin-project/lotus/build.NetworkBundle=calibrationnet
 calibnet: build-devnets
 
 butterflynet: GOFLAGS+=-tags=butterflynet
+butterflynet: GOFLAGS+=-ldflags=-X=github.com/filecoin-project/lotus/build.NetworkBundle=butterflynet
 butterflynet: build-devnets
 
 interopnet: GOFLAGS+=-tags=interopnet
+interopnet: GOFLAGS+=-ldflags=-X=github.com/filecoin-project/lotus/build.NetworkBundle=caterpillarnet
 interopnet: build-devnets
 
 lotus: $(BUILD_DEPS)

build/builtin_actors.go

@@ -0,0 +1,27 @@
+package build
+
+import (
+	"bytes"
+
+	"github.com/filecoin-project/lotus/chain/actors"
+
+	"github.com/BurntSushi/toml"
+)
+
+var BuiltinActorReleases map[actors.Version]string
+
+func init() {
+	BuiltinActorReleases = make(map[actors.Version]string)
+
+	spec := BundleSpec{}
+
+	r := bytes.NewReader(BuiltinActorBundles)
+	_, err := toml.DecodeReader(r, &spec)
+	if err != nil {
+		panic(err)
+	}
+
+	for _, b := range spec.Bundles {
+		BuiltinActorReleases[b.Version] = b.Release
+	}
+}

build/bundle.go

@@ -0,0 +1,21 @@
+package build
+
+import (
+	_ "embed"
+
+	"github.com/filecoin-project/lotus/chain/actors"
+)
+
+var NetworkBundle string
+
+//go:embed bundles.toml
+var BuiltinActorBundles []byte
+
+type BundleSpec struct {
+	Bundles []Bundle
+}
+
+type Bundle struct {
+	Version actors.Version
+	Release string
+}

build/bundles.toml

@@ -0,0 +1,3 @@
+[[bundles]]
+version = 8
+release = "b71c2ec785aec23d"

chain/actors/manifest.go

@@ -130,12 +130,3 @@ func LoadBundle(ctx context.Context, bs blockstore.Blockstore, av Version, data
 
 	return nil
 }
-
-func LoadManifestFromBundle(ctx context.Context, bs blockstore.Blockstore, av Version, data []byte) error {
-	if err := LoadBundle(ctx, bs, av, data); err != nil {
-		return err
-	}
-
-	cborStore := cbor.NewCborStore(bs)
-	return LoadManifests(ctx, cborStore)
-}

cli/init.go

@@ -2,21 +2,20 @@ package cli
 
 import (
 	"context"
-	"fmt"
 
 	"github.com/filecoin-project/lotus/blockstore"
 	"github.com/filecoin-project/lotus/build"
-	"github.com/filecoin-project/lotus/chain/actors"
+	"github.com/filecoin-project/lotus/node/bundle"
 )
 
 func init() {
 	// preload manifest so that we have the correct code CID inventory for cli since that doesn't
 	// go through CI
-	if len(build.BuiltinActorsV8Bundle()) > 0 {
-		bs := blockstore.NewMemory()
+	// TODO loading the bundle in every cli invocation adds some latency; we should figure out a way
+	//      to load actor CIDs without incurring this hit.
+	bs := blockstore.NewMemory()
 
-		if err := actors.LoadManifestFromBundle(context.TODO(), bs, actors.Version8, build.BuiltinActorsV8Bundle()); err != nil {
-			panic(fmt.Errorf("error loading actor manifest: %w", err))
-		}
+	if err := bundle.FetchAndLoadBundles(context.Background(), bs, build.BuiltinActorReleases); err != nil {
+		panic(err)
 	}
 }