diff --git a/api/api_storage.go b/api/api_storage.go index 5785ede1fdd..9de54ff3e04 100644 --- a/api/api_storage.go +++ b/api/api_storage.go @@ -227,6 +227,9 @@ type StorageMiner interface { // DagstoreGC runs garbage collection on the DAG store. DagstoreGC(ctx context.Context) ([]DagstoreShardResult, error) //perm:admin + // DagstoreRegisterShard registers a shard manually with dagstore with given pieceCID + DagstoreRegisterShard(ctx context.Context, key string) error //perm:admin + // IndexerAnnounceDeal informs indexer nodes that a new deal was received, // so they can download its index IndexerAnnounceDeal(ctx context.Context, proposalCid cid.Cid) error //perm:admin diff --git a/api/proxy_gen.go b/api/proxy_gen.go index 7542ac36726..a40d665ce3d 100644 --- a/api/proxy_gen.go +++ b/api/proxy_gen.go @@ -663,6 +663,8 @@ type StorageMinerStruct struct { DagstoreRecoverShard func(p0 context.Context, p1 string) error `perm:"write"` + DagstoreRegisterShard func(p0 context.Context, p1 string) error `perm:"admin"` + DealsConsiderOfflineRetrievalDeals func(p0 context.Context) (bool, error) `perm:"admin"` DealsConsiderOfflineStorageDeals func(p0 context.Context) (bool, error) `perm:"admin"` @@ -3990,6 +3992,17 @@ func (s *StorageMinerStub) DagstoreRecoverShard(p0 context.Context, p1 string) e return ErrNotSupported } +func (s *StorageMinerStruct) DagstoreRegisterShard(p0 context.Context, p1 string) error { + if s.Internal.DagstoreRegisterShard == nil { + return ErrNotSupported + } + return s.Internal.DagstoreRegisterShard(p0, p1) +} + +func (s *StorageMinerStub) DagstoreRegisterShard(p0 context.Context, p1 string) error { + return ErrNotSupported +} + func (s *StorageMinerStruct) DealsConsiderOfflineRetrievalDeals(p0 context.Context) (bool, error) { if s.Internal.DealsConsiderOfflineRetrievalDeals == nil { return false, ErrNotSupported diff --git a/build/openrpc/full.json.gz b/build/openrpc/full.json.gz index 92731a5f6c7..5b8d28f20c1 100644 Binary files a/build/openrpc/full.json.gz and b/build/openrpc/full.json.gz differ diff --git a/build/openrpc/miner.json.gz b/build/openrpc/miner.json.gz index 43b182dfa58..c409344e140 100644 Binary files a/build/openrpc/miner.json.gz and b/build/openrpc/miner.json.gz differ diff --git a/build/openrpc/worker.json.gz b/build/openrpc/worker.json.gz index cbbeec7d1e4..8e5a736f04a 100644 Binary files a/build/openrpc/worker.json.gz and b/build/openrpc/worker.json.gz differ diff --git a/cmd/lotus-miner/dagstore.go b/cmd/lotus-miner/dagstore.go index b35dcbe5037..37d0488259e 100644 --- a/cmd/lotus-miner/dagstore.go +++ b/cmd/lotus-miner/dagstore.go @@ -19,6 +19,7 @@ var dagstoreCmd = &cli.Command{ Usage: "Manage the dagstore on the markets subsystem", Subcommands: []*cli.Command{ dagstoreListShardsCmd, + dagstoreRegisterShardCmd, dagstoreInitializeShardCmd, dagstoreRecoverShardCmd, dagstoreInitializeAllCmd, @@ -59,6 +60,45 @@ var dagstoreListShardsCmd = &cli.Command{ }, } +var dagstoreRegisterShardCmd = &cli.Command{ + Name: "register-shard", + ArgsUsage: "[key]", + Usage: "Register a shard", + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "color", + Usage: "use color in display output", + DefaultText: "depends on output being a TTY", + }, + }, + Action: func(cctx *cli.Context) error { + if cctx.IsSet("color") { + color.NoColor = !cctx.Bool("color") + } + + if cctx.NArg() != 1 { + return fmt.Errorf("must provide a single shard key") + } + + marketsAPI, closer, err := lcli.GetMarketsAPI(cctx) + if err != nil { + return err + } + defer closer() + + ctx := lcli.ReqContext(cctx) + + shardKey := cctx.Args().First() + err = marketsAPI.DagstoreRegisterShard(ctx, shardKey) + if err != nil { + return err + } + + fmt.Println("Registered shard " + shardKey) + return nil + }, +} + var dagstoreInitializeShardCmd = &cli.Command{ Name: "initialize-shard", ArgsUsage: "[key]", diff --git a/documentation/en/api-v0-methods-miner.md b/documentation/en/api-v0-methods-miner.md index 7642efbee5a..82fe2c3d9da 100644 --- a/documentation/en/api-v0-methods-miner.md +++ b/documentation/en/api-v0-methods-miner.md @@ -27,6 +27,7 @@ * [DagstoreListShards](#DagstoreListShards) * [DagstoreLookupPieces](#DagstoreLookupPieces) * [DagstoreRecoverShard](#DagstoreRecoverShard) + * [DagstoreRegisterShard](#DagstoreRegisterShard) * [Deals](#Deals) * [DealsConsiderOfflineRetrievalDeals](#DealsConsiderOfflineRetrievalDeals) * [DealsConsiderOfflineStorageDeals](#DealsConsiderOfflineStorageDeals) @@ -639,6 +640,21 @@ Inputs: Response: `{}` +### DagstoreRegisterShard +DagstoreRegisterShard registers a shard manually with dagstore with given pieceCID + + +Perms: admin + +Inputs: +```json +[ + "string value" +] +``` + +Response: `{}` + ## Deals diff --git a/documentation/en/cli-lotus-miner.md b/documentation/en/cli-lotus-miner.md index 5be59a1b67f..e03ae7cc131 100644 --- a/documentation/en/cli-lotus-miner.md +++ b/documentation/en/cli-lotus-miner.md @@ -1035,6 +1035,7 @@ USAGE: COMMANDS: list-shards List all shards known to the dagstore, with their current status + register-shard Register a shard initialize-shard Initialize the specified shard recover-shard Attempt to recover a shard in errored state initialize-all Initialize all uninitialized shards, streaming results as they're produced; only shards for unsealed pieces are initialized by default @@ -1061,6 +1062,20 @@ OPTIONS: ``` +### lotus-miner dagstore register-shard +``` +NAME: + lotus-miner dagstore register-shard - Register a shard + +USAGE: + lotus-miner dagstore register-shard [command options] [key] + +OPTIONS: + --color use color in display output (default: depends on output being a TTY) + --help, -h show help (default: false) + +``` + ### lotus-miner dagstore initialize-shard ``` NAME: diff --git a/node/impl/storminer.go b/node/impl/storminer.go index f34761d891a..eb895200464 100644 --- a/node/impl/storminer.go +++ b/node/impl/storminer.go @@ -37,8 +37,10 @@ import ( "github.com/filecoin-project/go-fil-markets/piecestore" "github.com/filecoin-project/go-fil-markets/retrievalmarket" "github.com/filecoin-project/go-fil-markets/storagemarket" + filmktsstore "github.com/filecoin-project/go-fil-markets/stores" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/network" + mktsdagstore "github.com/filecoin-project/lotus/markets/dagstore" sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage" "github.com/filecoin-project/lotus/extern/sector-storage/fsutil" @@ -83,6 +85,7 @@ type StorageMinerAPI struct { SectorBlocks *sectorblocks.SectorBlocks `optional:"true"` Host host.Host `optional:"true"` DAGStore *dagstore.DAGStore `optional:"true"` + DAGStoreWrapper *mktsdagstore.Wrapper `optional:"true"` // Miner / storage Miner *storage.Miner `optional:"true"` @@ -792,6 +795,35 @@ func (sm *StorageMinerAPI) DagstoreListShards(ctx context.Context) ([]api.Dagsto return ret, nil } +func (sm *StorageMinerAPI) DagstoreRegisterShard(ctx context.Context, key string) error { + if sm.DAGStore == nil { + return fmt.Errorf("dagstore not available on this node") + } + + // First check if the shard has already been registered + k := shard.KeyFromString(key) + _, err := sm.DAGStore.GetShardInfo(k) + if err == nil { + // Shard already registered, nothing further to do + return nil + } + // If the shard is not registered we would expect ErrShardUnknown + if !errors.Is(err, dagstore.ErrShardUnknown) { + return fmt.Errorf("getting shard info from DAG store: %w", err) + } + + pieceCid, err := cid.Parse(key) + if err != nil { + return fmt.Errorf("parsing shard key as piece cid: %w", err) + } + + if err = filmktsstore.RegisterShardSync(ctx, sm.DAGStoreWrapper, pieceCid, "", true); err != nil { + return fmt.Errorf("failed to register shard: %w", err) + } + + return nil +} + func (sm *StorageMinerAPI) DagstoreInitializeShard(ctx context.Context, key string) error { if sm.DAGStore == nil { return fmt.Errorf("dagstore not available on this node")