From 79e34bca847cfc911e3711aeae4c8eeb711d3d25 Mon Sep 17 00:00:00 2001 From: gammazero Date: Tue, 15 Mar 2022 02:52:21 -0700 Subject: [PATCH 1/3] Use new metadata api with multi-protocol support --- cardatatransfer/cardatatransfer.go | 4 +- cmd/go.mod | 6 +- cmd/go.sum | 24 +++-- cmd/provider/find.go | 42 ++++++-- cmd/provider/flags.go | 13 +++ e2e_retrieve_test.go | 10 +- engine/engine.go | 32 +++--- engine/engine_test.go | 1 + engine/example_test.go | 5 +- engine/linksystem_test.go | 11 +- go.mod | 7 +- go.sum | 24 +++-- interface.go | 2 +- metadata/bitswap.go | 42 +++++++- metadata/graphsync_filecoinv1.go | 113 +++++++++++++++----- metadata/graphsync_filecoinv1_test.go | 12 +-- mock/interface.go | 2 +- server/admin/http/importcar_handler.go | 10 +- server/admin/http/importcar_handler_test.go | 15 ++- supplier/car_supplier.go | 2 +- supplier/car_supplier_test.go | 20 ++-- 21 files changed, 289 insertions(+), 108 deletions(-) diff --git a/cardatatransfer/cardatatransfer.go b/cardatatransfer/cardatatransfer.go index 9dd226e2..ec54a12a 100644 --- a/cardatatransfer/cardatatransfer.go +++ b/cardatatransfer/cardatatransfer.go @@ -64,7 +64,7 @@ func StartCarDataTransfer(dt datatransfer.Manager, supplier BlockStoreSupplier) return nil } -func MetadataFromContextID(contextID []byte) (stiapi.Metadata, error) { +func MetadataFromContextID(contextID []byte) (stiapi.ParsedMetadata, error) { pieceCid, err := cid.Prefix{ Version: 1, Codec: uint64(multicodec.TransportGraphsyncFilecoinv1), @@ -72,7 +72,7 @@ func MetadataFromContextID(contextID []byte) (stiapi.Metadata, error) { MhLength: -1, }.Sum(contextID) if err != nil { - return stiapi.Metadata{}, err + return stiapi.ParsedMetadata{}, err } filecoinV1Metadata := &metadata.GraphsyncFilecoinV1Metadata{ PieceCID: pieceCid, diff --git a/cmd/go.mod b/cmd/go.mod index 4278e206..800f364a 100644 --- a/cmd/go.mod +++ b/cmd/go.mod @@ -4,9 +4,9 @@ go 1.16 require ( github.com/filecoin-project/go-data-transfer v1.14.0 - github.com/filecoin-project/go-legs v0.3.8 + github.com/filecoin-project/go-legs v0.3.9 github.com/filecoin-project/index-provider v0.2.1 - github.com/filecoin-project/storetheindex v0.3.5 + github.com/filecoin-project/storetheindex v0.3.7-0.20220309174359-164d0cba7ea6 github.com/ipfs/go-cid v0.1.0 github.com/ipfs/go-datastore v0.5.1 github.com/ipfs/go-ds-leveldb v0.5.0 @@ -14,7 +14,7 @@ require ( github.com/ipfs/go-ipfs v0.11.0 github.com/ipfs/go-log/v2 v2.5.0 github.com/ipld/go-car/v2 v2.1.1 - github.com/ipld/go-ipld-prime v0.14.4 + github.com/ipld/go-ipld-prime v0.16.0 github.com/libp2p/go-libp2p v0.18.0-rc3 github.com/libp2p/go-libp2p-core v0.14.0 github.com/mitchellh/go-homedir v1.1.0 diff --git a/cmd/go.sum b/cmd/go.sum index bc1c012a..d5dd3028 100644 --- a/cmd/go.sum +++ b/cmd/go.sum @@ -212,10 +212,10 @@ github.com/filecoin-project/go-data-transfer v1.14.0/go.mod h1:wNJKhaLLYBJDM3VFv github.com/filecoin-project/go-ds-versioning v0.0.0-20211206185234-508abd7c2aff/go.mod h1:C9/l9PnB1+mwPa26BBVpCjG/XQCB0yj/q5CK2J8X1I4= github.com/filecoin-project/go-ds-versioning v0.1.1 h1:JiyBqaQlwC+UM0WhcBtVEeT3XrX59mQhT8U3p7nu86o= github.com/filecoin-project/go-ds-versioning v0.1.1/go.mod h1:C9/l9PnB1+mwPa26BBVpCjG/XQCB0yj/q5CK2J8X1I4= -github.com/filecoin-project/go-indexer-core v0.2.8/go.mod h1:IagNfTdFuX4057kla43PjRCn3yBuUiZgIxuA0hTUamY= -github.com/filecoin-project/go-legs v0.3.7/go.mod h1:pgekGm8/gKY5zCtQ/qGAoSjGP92wTLFqpO3GPHeu8YU= -github.com/filecoin-project/go-legs v0.3.8 h1:HDhunjC+E5g3s47zUQY/YnDveatOXWJyfK519jdKhOA= +github.com/filecoin-project/go-indexer-core v0.2.9/go.mod h1:u03I3HB6ZnqCc3cm8Tq+QkTWBbfKOvNxM8K6Ny/IHRw= github.com/filecoin-project/go-legs v0.3.8/go.mod h1:FroH5LQUTfYzs8huA4PjW8spDF691uvK8uk9YAfDa78= +github.com/filecoin-project/go-legs v0.3.9 h1:YDW+nHn3fdl/5M7dE0EOxU8E3EgJGjuccLcz9xaT3kg= +github.com/filecoin-project/go-legs v0.3.9/go.mod h1:FroH5LQUTfYzs8huA4PjW8spDF691uvK8uk9YAfDa78= github.com/filecoin-project/go-state-types v0.1.0 h1:9r2HCSMMCmyMfGyMKxQtv0GKp6VT/m5GgVk8EhYbLJU= github.com/filecoin-project/go-state-types v0.1.0/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g= github.com/filecoin-project/go-statemachine v0.0.0-20200925024713-05bd7c71fbfe h1:dF8u+LEWeIcTcfUcCf3WFVlc81Fr2JKg8zPzIbBDKDw= @@ -223,8 +223,8 @@ github.com/filecoin-project/go-statemachine v0.0.0-20200925024713-05bd7c71fbfe/g github.com/filecoin-project/go-statestore v0.1.0/go.mod h1:LFc9hD+fRxPqiHiaqUEZOinUJB4WARkRfNl10O7kTnI= github.com/filecoin-project/go-statestore v0.2.0 h1:cRRO0aPLrxKQCZ2UOQbzFGn4WDNdofHZoGPjfNaAo5Q= github.com/filecoin-project/go-statestore v0.2.0/go.mod h1:8sjBYbS35HwPzct7iT4lIXjLlYyPor80aU7t7a/Kspo= -github.com/filecoin-project/storetheindex v0.3.5 h1:KoS9TvjPm6zIZfUH8atAHJbVHOO7GTP1MdTG+v0eE+Q= -github.com/filecoin-project/storetheindex v0.3.5/go.mod h1:0r3d0kSpK63O6AvLr1CjAINLi+nWD49clzcnKV+GLpI= +github.com/filecoin-project/storetheindex v0.3.7-0.20220309174359-164d0cba7ea6 h1:Pg3e+uEgTNPHxhWTl+ouhcnkRpUkgL1UJpo5ekuVwF4= +github.com/filecoin-project/storetheindex v0.3.7-0.20220309174359-164d0cba7ea6/go.mod h1:5QoLWR+55xZXuJA7Y4eJQTbryEo5p6gvIPUqtXK5eaY= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6/go.mod h1:1i71OnUq3iUe1ma7Lr6yG6/rjvM3emb6yoL7xLFzcVQ= github.com/flynn/noise v1.0.0 h1:DlTHqmzmvcEiKj+4RYo/imoswx/4r6iBlCMfVtrMXpQ= @@ -234,8 +234,9 @@ github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiD github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= -github.com/frankban/quicktest v1.14.0 h1:+cqqvzZV87b4adx/5ayVOaYZ2CrvM4ejQvUdBzPPUss= github.com/frankban/quicktest v1.14.0/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09mUdL/ijj8og= +github.com/frankban/quicktest v1.14.2 h1:SPb1KFFmM+ybpEjPUhCCkZOM5xlovT5UbrMvWnXyBns= +github.com/frankban/quicktest v1.14.2/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI= @@ -330,8 +331,9 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o= +github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -630,12 +632,13 @@ github.com/ipld/go-ipld-prime v0.12.3/go.mod h1:PaeLYq8k6dJLmDUSLrzkEpoGV4PEfe/1 github.com/ipld/go-ipld-prime v0.14.0/go.mod h1:9ASQLwUFLptCov6lIYc70GRB4V7UTyLD0IJtrDJe6ZM= github.com/ipld/go-ipld-prime v0.14.1/go.mod h1:QcE4Y9n/ZZr8Ijg5bGPT0GqYWgZ1704nH0RDcQtgTP0= github.com/ipld/go-ipld-prime v0.14.2/go.mod h1:QcE4Y9n/ZZr8Ijg5bGPT0GqYWgZ1704nH0RDcQtgTP0= -github.com/ipld/go-ipld-prime v0.14.4 h1:bqhmume8+nbNsX4/+J6eohktfZHAI8GKrF3rQ0xgOyc= github.com/ipld/go-ipld-prime v0.14.4/go.mod h1:QcE4Y9n/ZZr8Ijg5bGPT0GqYWgZ1704nH0RDcQtgTP0= +github.com/ipld/go-ipld-prime v0.16.0 h1:RS5hhjB/mcpeEPJvfyj0qbOj/QL+/j05heZ0qa97dVo= +github.com/ipld/go-ipld-prime v0.16.0/go.mod h1:axSCuOCBPqrH+gvXr2w9uAOulJqBPhHPT2PjoiiU1qA= github.com/ipld/go-ipld-prime-proto v0.0.0-20191113031812-e32bd156a1e5/go.mod h1:gcvzoEDBjwycpXt3LBE061wT9f46szXGHAmj9uoP6fU= github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20211210234204-ce2a1c70cd73 h1:TsyATB2ZRRQGTwafJdgEUQkmjOExRV0DNokcihZxbnQ= github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20211210234204-ce2a1c70cd73/go.mod h1:2PJ0JgxyB08t0b2WKrcuqI3di0V+5n6RS/LTUJhkoxY= -github.com/ipld/go-storethehash v0.0.1/go.mod h1:w8cQfWInks8lvvbQTiKbCPusU9v0sqiViBihTHbavpQ= +github.com/ipld/go-storethehash v0.0.2/go.mod h1:w8cQfWInks8lvvbQTiKbCPusU9v0sqiViBihTHbavpQ= github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52 h1:QG4CGBqCeuBo6aZlGAamSkxWdgWfZGeE49eUOWJPA4c= github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52/go.mod h1:fdg+/X9Gg4AsAIzWpEHwnqd+QY3b7lajxyjE1m4hkq4= github.com/jackpal/gateway v1.0.5/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA= @@ -1462,7 +1465,10 @@ github.com/whyrusleeping/cbor-gen v0.0.0-20200810223238-211df3b9e24c/go.mod h1:f github.com/whyrusleeping/cbor-gen v0.0.0-20200812213548-958ddffe352c/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= github.com/whyrusleeping/cbor-gen v0.0.0-20200826160007-0b9f6c5fb163/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= github.com/whyrusleeping/cbor-gen v0.0.0-20210219115102-f37d292932f2/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= +<<<<<<< HEAD github.com/whyrusleeping/cbor-gen v0.0.0-20220224212727-7a699437a831/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= +======= +>>>>>>> Use new metadata api with multi-protocol support github.com/whyrusleeping/cbor-gen v0.0.0-20220302191723-37c43cae8e14 h1:vo2wkP2ceHyGyZwFFtAabpot03EeSxxwAe57pOI9E/4= github.com/whyrusleeping/cbor-gen v0.0.0-20220302191723-37c43cae8e14/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f h1:jQa4QT2UP9WYv2nzyawpKMOCl+Z/jW7djv2/J50lj9E= diff --git a/cmd/provider/find.go b/cmd/provider/find.go index af84ee0e..97df3bb0 100644 --- a/cmd/provider/find.go +++ b/cmd/provider/find.go @@ -4,8 +4,11 @@ import ( "encoding/base64" "fmt" - httpfinderclient "github.com/filecoin-project/storetheindex/api/v0/finder/client/http" + "github.com/filecoin-project/storetheindex/api/v0/finder/client" + httpclient "github.com/filecoin-project/storetheindex/api/v0/finder/client/http" + p2pclient "github.com/filecoin-project/storetheindex/api/v0/finder/client/libp2p" "github.com/ipfs/go-cid" + "github.com/libp2p/go-libp2p-core/peer" "github.com/multiformats/go-multihash" "github.com/urfave/cli/v2" ) @@ -18,10 +21,7 @@ var FindCmd = &cli.Command{ } func findCommand(cctx *cli.Context) error { - cli, err := httpfinderclient.New(cctx.String("indexer")) - if err != nil { - return err - } + protocol := cctx.String("protocol") mhArgs := cctx.StringSlice("mh") cidArgs := cctx.StringSlice("cid") @@ -41,7 +41,36 @@ func findCommand(cctx *cli.Context) error { mhs = append(mhs, c.Hash()) } - resp, err := cli.FindBatch(cctx.Context, mhs) + var cl client.Finder + var err error + + switch protocol { + case "http": + cl, err = httpclient.New(cctx.String("indexer")) + if err != nil { + return err + } + case "libp2p": + peerID, err := peer.Decode(cctx.String("peerid")) + if err != nil { + return err + } + + c, err := p2pclient.New(nil, peerID) + if err != nil { + return err + } + + err = c.Connect(cctx.Context, cctx.String("indexer")) + if err != nil { + return err + } + cl = c + default: + return fmt.Errorf("unrecognized protocol type for client interaction: %s", protocol) + } + + resp, err := cl.FindBatch(cctx.Context, mhs) if err != nil { return err } @@ -57,7 +86,6 @@ func findCommand(cctx *cli.Context) error { for _, pr := range resp.MultihashResults[i].ProviderResults { fmt.Println(" Provider:", pr.Provider) fmt.Println(" ContextID:", base64.StdEncoding.EncodeToString(pr.ContextID)) - fmt.Println(" Proto:", pr.Metadata.ProtocolID) fmt.Println(" Metadata:", base64.StdEncoding.EncodeToString(pr.Metadata.Data)) } } diff --git a/cmd/provider/flags.go b/cmd/provider/flags.go index 95ef68b1..6e9fc5cf 100644 --- a/cmd/provider/flags.go +++ b/cmd/provider/flags.go @@ -34,6 +34,12 @@ var indexerFlag = &cli.StringFlag{ Required: true, } +var indexerIDFlag = &cli.StringFlag{ + Name: "peerid", + Usage: "Peer ID of indexer to use, when using libp2p", + Required: false, +} + var addrFlag = &cli.StringSliceFlag{ Name: "addr", Usage: `Provider address as multiaddr string, example: "/ip4/127.0.0.1/tcp/3103"`, @@ -43,6 +49,7 @@ var addrFlag = &cli.StringSliceFlag{ var findFlags = []cli.Flag{ indexerFlag, + indexerIDFlag, &cli.StringSliceFlag{ Name: "mh", Usage: "Specify multihash to use as indexer key, multiple OK", @@ -53,6 +60,12 @@ var findFlags = []cli.Flag{ Usage: "Specify CID to use as indexer key, multiple OK", Required: false, }, + &cli.StringFlag{ + Name: "protocol", + Usage: "Protocol to query the indexer (http, libp2p currently supported)", + Value: "http", + Required: false, + }, } var indexFlags = []cli.Flag{ diff --git a/e2e_retrieve_test.go b/e2e_retrieve_test.go index 79144337..2f332d70 100644 --- a/e2e_retrieve_test.go +++ b/e2e_retrieve_test.go @@ -130,10 +130,12 @@ func testRetrievalRoundTripWithTestCase(t *testing.T, tc testCase) { mdb, err := adv.FieldMetadata().AsBytes() require.NoError(t, err) - var receivedMd stiapi.Metadata + cids := testutil.GenerateCids(1) + pieceCID := cids[0] + receivedMd := stiapi.ParsedMetadata{Protocols: []stiapi.ProtocolMetadata{&metadata.GraphsyncFilecoinV1Metadata{PieceCID: pieceCID}}} err = receivedMd.UnmarshalBinary(mdb) require.NoError(t, err) - dtm := &metadata.GraphsyncFilecoinV1Metadata{} + dtm := metadata.GraphsyncFilecoinV1Metadata{} err = dtm.FromIndexerMetadata(receivedMd) require.NoError(t, err) @@ -209,7 +211,7 @@ func testReimportCarWtihTestCase(t *testing.T, tc testCase) { mdb, err := adv.FieldMetadata().AsBytes() require.NoError(t, err) - var receivedMd stiapi.Metadata + var receivedMd stiapi.ParsedMetadata err = receivedMd.UnmarshalBinary(mdb) require.NoError(t, err) @@ -242,7 +244,7 @@ func testReimportCarWtihTestCase(t *testing.T, tc testCase) { mdb2, err := adv2.FieldMetadata().AsBytes() require.NoError(t, err) - var receivedMd2 stiapi.Metadata + var receivedMd2 stiapi.ParsedMetadata err = receivedMd2.UnmarshalBinary(mdb2) require.NoError(t, err) diff --git a/engine/engine.go b/engine/engine.go index c917e946..b8baf956 100644 --- a/engine/engine.go +++ b/engine/engine.go @@ -11,6 +11,7 @@ import ( "github.com/filecoin-project/go-legs/httpsync" provider "github.com/filecoin-project/index-provider" "github.com/filecoin-project/index-provider/engine/chunker" + md "github.com/filecoin-project/index-provider/metadata" stiapi "github.com/filecoin-project/storetheindex/api/v0" "github.com/filecoin-project/storetheindex/api/v0/ingest/schema" "github.com/hashicorp/go-multierror" @@ -20,7 +21,6 @@ import ( logging "github.com/ipfs/go-log/v2" "github.com/ipld/go-ipld-prime" cidlink "github.com/ipld/go-ipld-prime/linking/cid" - "github.com/multiformats/go-multicodec" ) const ( @@ -238,7 +238,7 @@ func (e *Engine) RegisterMultihashLister(mhl provider.MultihashLister) { // Note that prior to calling this function a provider.MultihashLister must be registered. // // See: Engine.RegisterMultihashLister, Engine.Publish. -func (e *Engine) NotifyPut(ctx context.Context, contextID []byte, metadata stiapi.Metadata) (cid.Cid, error) { +func (e *Engine) NotifyPut(ctx context.Context, contextID []byte, metadata stiapi.ParsedMetadata) (cid.Cid, error) { // The multihash lister must have been registered for the linkSystem to know how to // go from contextID to list of CIDs. return e.publishAdvForIndex(ctx, contextID, metadata, false) @@ -251,7 +251,7 @@ func (e *Engine) NotifyPut(ctx context.Context, contextID []byte, metadata stiap // // See: Engine.RegisterMultihashLister, Engine.Publish. func (e *Engine) NotifyRemove(ctx context.Context, contextID []byte) (cid.Cid, error) { - return e.publishAdvForIndex(ctx, contextID, stiapi.Metadata{}, true) + return e.publishAdvForIndex(ctx, contextID, stiapi.ParsedMetadata{}, true) } // Shutdown shuts down the engine and discards all resources opened by the engine. @@ -312,7 +312,7 @@ func (e *Engine) GetLatestAdv(ctx context.Context) (cid.Cid, schema.Advertisemen return latestAdCid, ad, nil } -func (e *Engine) publishAdvForIndex(ctx context.Context, contextID []byte, metadata stiapi.Metadata, isRm bool) (cid.Cid, error) { +func (e *Engine) publishAdvForIndex(ctx context.Context, contextID []byte, metadata stiapi.ParsedMetadata, isRm bool) (cid.Cid, error) { var err error var cidsLnk cidlink.Link @@ -408,9 +408,7 @@ func (e *Engine) publishAdvForIndex(ctx context.Context, contextID []byte, metad // The advertisement still requires a valid metadata even though // metadata is not used for removal. Create a valid empty metadata. - metadata = stiapi.Metadata{ - ProtocolID: multicodec.TransportGraphsyncFilecoinv1, - } + metadata = stiapi.ParsedMetadata{Protocols: []stiapi.ProtocolMetadata{&md.GraphsyncFilecoinV1Metadata{}}} } // Get the previous advertisement that was generated @@ -433,8 +431,12 @@ func (e *Engine) publishAdvForIndex(ctx context.Context, contextID []byte, metad previousLnk = nb.Build().(schema.Link_Advertisement) } + encMeta, err := metadata.MarshalBinary() + if err != nil { + return cid.Undef, fmt.Errorf("failed to marshal metadata: %w", err) + } adv, err := schema.NewAdvertisement(e.key, previousLnk, cidsLnk, - contextID, metadata, isRm, e.h.ID().String(), e.retrievalAddrsAsString()) + contextID, stiapi.Metadata(encMeta), isRm, e.h.ID().String(), e.retrievalAddrsAsString()) if err != nil { return cid.Undef, fmt.Errorf("failed to create advertisement: %s", err) } @@ -474,7 +476,7 @@ func (e *Engine) getCidKeyMap(ctx context.Context, c cid.Cid) ([]byte, error) { return e.ds.Get(ctx, datastore.NewKey(cidToKeyMapPrefix+c.String())) } -func (e *Engine) putKeyMetadataMap(ctx context.Context, contextID []byte, metadata stiapi.Metadata) error { +func (e *Engine) putKeyMetadataMap(ctx context.Context, contextID []byte, metadata stiapi.ParsedMetadata) error { data, err := metadata.MarshalBinary() if err != nil { return err @@ -482,16 +484,16 @@ func (e *Engine) putKeyMetadataMap(ctx context.Context, contextID []byte, metada return e.ds.Put(ctx, datastore.NewKey(keyToMetadataMapPrefix+string(contextID)), data) } -func (e *Engine) getKeyMetadataMap(ctx context.Context, contextID []byte) (stiapi.Metadata, error) { +func (e *Engine) getKeyMetadataMap(ctx context.Context, contextID []byte) (stiapi.ParsedMetadata, error) { data, err := e.ds.Get(ctx, datastore.NewKey(keyToMetadataMapPrefix+string(contextID))) if err != nil { - return stiapi.Metadata{}, err + return stiapi.ParsedMetadata{}, err } - var metadata stiapi.Metadata - if err = metadata.UnmarshalBinary(data); err != nil { - return stiapi.Metadata{}, err + m := stiapi.ParsedMetadata{} + if err := m.UnmarshalBinary(data); err != nil { + return stiapi.ParsedMetadata{}, err } - return metadata, nil + return m, nil } func (e *Engine) deleteKeyMetadataMap(ctx context.Context, contextID []byte) error { diff --git a/engine/engine_test.go b/engine/engine_test.go index 5f27ba0c..a7c18fec 100644 --- a/engine/engine_test.go +++ b/engine/engine_test.go @@ -16,6 +16,7 @@ import ( "github.com/filecoin-project/index-provider/metadata" "github.com/filecoin-project/index-provider/testutil" "github.com/filecoin-project/storetheindex/api/v0/ingest/schema" + v0util "github.com/filecoin-project/storetheindex/api/v0/util" "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore" dssync "github.com/ipfs/go-datastore/sync" diff --git a/engine/example_test.go b/engine/example_test.go index 067c1df1..50a9eade 100644 --- a/engine/example_test.go +++ b/engine/example_test.go @@ -8,6 +8,7 @@ import ( provider "github.com/filecoin-project/index-provider" "github.com/filecoin-project/index-provider/engine" "github.com/filecoin-project/index-provider/metadata" + "github.com/filecoin-project/storetheindex/api/v0" "github.com/libp2p/go-libp2p" "github.com/multiformats/go-multihash" ) @@ -59,9 +60,11 @@ func Example_advertiseHelloWorld() { } fmt.Println("✓ Provider engine started.") + parsedMetadata := v0.ParsedMetadata{Protocols: []v0.ProtocolMetadata{&metadata.BitswapMetadata{}}} + // Note that this example publishes an ad with bitswap metadata as an example. // But it does not instantiate a bitswap server to serve retrievals. - adCid, err := engine.NotifyPut(context.Background(), []byte(sayHelloCtxID), metadata.BitswapMetadata) + adCid, err := engine.NotifyPut(context.Background(), []byte(sayHelloCtxID), parsedMetadata) if err != nil { panic(err) } diff --git a/engine/linksystem_test.go b/engine/linksystem_test.go index 695e2a15..7c5f2b68 100644 --- a/engine/linksystem_test.go +++ b/engine/linksystem_test.go @@ -12,6 +12,7 @@ import ( "github.com/filecoin-project/index-provider/engine/chunker" "github.com/filecoin-project/index-provider/metadata" "github.com/filecoin-project/index-provider/testutil" + "github.com/filecoin-project/storetheindex/api/v0" "github.com/filecoin-project/storetheindex/api/v0/ingest/schema" "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore" @@ -59,8 +60,10 @@ func Test_RemovalAdvertisementWithNoEntriesIsRetrievable(t *testing.T) { return nil, errors.New("not found") }) + parsedMetadata := v0.ParsedMetadata{Protocols: []v0.ProtocolMetadata{&metadata.GraphsyncFilecoinV1Metadata{}}} + // Publish content added advertisement. - adAddCid, err := subject.NotifyPut(ctx, ctxID, metadata.BitswapMetadata) + adAddCid, err := subject.NotifyPut(ctx, ctxID, parsedMetadata) require.NoError(t, err) adAdd, err := subject.GetAdv(ctx, adAddCid) require.NoError(t, err) @@ -132,7 +135,9 @@ func Test_EvictedCachedEntriesChainIsRegeneratedGracefully(t *testing.T) { return nil, errors.New("not found") }) - ad1Cid, err := subject.NotifyPut(ctx, ad1CtxID, metadata.BitswapMetadata) + parsedMetadata := v0.ParsedMetadata{Protocols: []v0.ProtocolMetadata{&metadata.GraphsyncFilecoinV1Metadata{}}} + + ad1Cid, err := subject.NotifyPut(ctx, ad1CtxID, parsedMetadata) require.NoError(t, err) ad1, err := subject.GetAdv(ctx, ad1Cid) require.NoError(t, err) @@ -142,7 +147,7 @@ func Test_EvictedCachedEntriesChainIsRegeneratedGracefully(t *testing.T) { requireChunkIsCached(t, subject.Chunker(), ad1EntriesChain...) a1Chunks := requireLoadEntryChunkFromEngine(t, subject, ad1EntriesChain...) - ad2Cid, err := subject.NotifyPut(ctx, ad2CtxID, metadata.BitswapMetadata) + ad2Cid, err := subject.NotifyPut(ctx, ad2CtxID, parsedMetadata) require.NoError(t, err) ad2, err := subject.GetAdv(ctx, ad2Cid) require.NoError(t, err) diff --git a/go.mod b/go.mod index 82f614ac..67f590cd 100644 --- a/go.mod +++ b/go.mod @@ -4,9 +4,9 @@ go 1.16 require ( github.com/filecoin-project/go-data-transfer v1.14.0 - github.com/filecoin-project/go-legs v0.3.8 + github.com/filecoin-project/go-legs v0.3.9 github.com/filecoin-project/go-state-types v0.1.0 - github.com/filecoin-project/storetheindex v0.3.5 + github.com/filecoin-project/storetheindex v0.3.7-0.20220309174359-164d0cba7ea6 github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da github.com/golang/mock v1.6.0 github.com/gorilla/mux v1.7.4 @@ -19,13 +19,14 @@ require ( github.com/ipfs/go-log/v2 v2.5.0 github.com/ipld/go-car/v2 v2.1.1 github.com/ipld/go-codec-dagpb v1.3.0 - github.com/ipld/go-ipld-prime v0.14.4 + github.com/ipld/go-ipld-prime v0.16.0 github.com/libp2p/go-libp2p v0.18.0-rc3 github.com/libp2p/go-libp2p-core v0.14.0 github.com/libp2p/go-libp2p-pubsub v0.6.1 github.com/multiformats/go-multiaddr v0.5.0 github.com/multiformats/go-multicodec v0.4.1 github.com/multiformats/go-multihash v0.1.0 + github.com/multiformats/go-varint v0.0.6 github.com/stretchr/testify v1.7.0 github.com/whyrusleeping/cbor-gen v0.0.0-20220302191723-37c43cae8e14 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 diff --git a/go.sum b/go.sum index 0653aba0..9a5135f6 100644 --- a/go.sum +++ b/go.sum @@ -212,10 +212,10 @@ github.com/filecoin-project/go-data-transfer v1.14.0/go.mod h1:wNJKhaLLYBJDM3VFv github.com/filecoin-project/go-ds-versioning v0.0.0-20211206185234-508abd7c2aff/go.mod h1:C9/l9PnB1+mwPa26BBVpCjG/XQCB0yj/q5CK2J8X1I4= github.com/filecoin-project/go-ds-versioning v0.1.1 h1:JiyBqaQlwC+UM0WhcBtVEeT3XrX59mQhT8U3p7nu86o= github.com/filecoin-project/go-ds-versioning v0.1.1/go.mod h1:C9/l9PnB1+mwPa26BBVpCjG/XQCB0yj/q5CK2J8X1I4= -github.com/filecoin-project/go-indexer-core v0.2.8/go.mod h1:IagNfTdFuX4057kla43PjRCn3yBuUiZgIxuA0hTUamY= -github.com/filecoin-project/go-legs v0.3.7/go.mod h1:pgekGm8/gKY5zCtQ/qGAoSjGP92wTLFqpO3GPHeu8YU= -github.com/filecoin-project/go-legs v0.3.8 h1:HDhunjC+E5g3s47zUQY/YnDveatOXWJyfK519jdKhOA= +github.com/filecoin-project/go-indexer-core v0.2.9/go.mod h1:u03I3HB6ZnqCc3cm8Tq+QkTWBbfKOvNxM8K6Ny/IHRw= github.com/filecoin-project/go-legs v0.3.8/go.mod h1:FroH5LQUTfYzs8huA4PjW8spDF691uvK8uk9YAfDa78= +github.com/filecoin-project/go-legs v0.3.9 h1:YDW+nHn3fdl/5M7dE0EOxU8E3EgJGjuccLcz9xaT3kg= +github.com/filecoin-project/go-legs v0.3.9/go.mod h1:FroH5LQUTfYzs8huA4PjW8spDF691uvK8uk9YAfDa78= github.com/filecoin-project/go-state-types v0.1.0 h1:9r2HCSMMCmyMfGyMKxQtv0GKp6VT/m5GgVk8EhYbLJU= github.com/filecoin-project/go-state-types v0.1.0/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g= github.com/filecoin-project/go-statemachine v0.0.0-20200925024713-05bd7c71fbfe h1:dF8u+LEWeIcTcfUcCf3WFVlc81Fr2JKg8zPzIbBDKDw= @@ -223,8 +223,8 @@ github.com/filecoin-project/go-statemachine v0.0.0-20200925024713-05bd7c71fbfe/g github.com/filecoin-project/go-statestore v0.1.0/go.mod h1:LFc9hD+fRxPqiHiaqUEZOinUJB4WARkRfNl10O7kTnI= github.com/filecoin-project/go-statestore v0.2.0 h1:cRRO0aPLrxKQCZ2UOQbzFGn4WDNdofHZoGPjfNaAo5Q= github.com/filecoin-project/go-statestore v0.2.0/go.mod h1:8sjBYbS35HwPzct7iT4lIXjLlYyPor80aU7t7a/Kspo= -github.com/filecoin-project/storetheindex v0.3.5 h1:KoS9TvjPm6zIZfUH8atAHJbVHOO7GTP1MdTG+v0eE+Q= -github.com/filecoin-project/storetheindex v0.3.5/go.mod h1:0r3d0kSpK63O6AvLr1CjAINLi+nWD49clzcnKV+GLpI= +github.com/filecoin-project/storetheindex v0.3.7-0.20220309174359-164d0cba7ea6 h1:Pg3e+uEgTNPHxhWTl+ouhcnkRpUkgL1UJpo5ekuVwF4= +github.com/filecoin-project/storetheindex v0.3.7-0.20220309174359-164d0cba7ea6/go.mod h1:5QoLWR+55xZXuJA7Y4eJQTbryEo5p6gvIPUqtXK5eaY= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6/go.mod h1:1i71OnUq3iUe1ma7Lr6yG6/rjvM3emb6yoL7xLFzcVQ= github.com/flynn/noise v1.0.0 h1:DlTHqmzmvcEiKj+4RYo/imoswx/4r6iBlCMfVtrMXpQ= @@ -234,8 +234,9 @@ github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiD github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= -github.com/frankban/quicktest v1.14.0 h1:+cqqvzZV87b4adx/5ayVOaYZ2CrvM4ejQvUdBzPPUss= github.com/frankban/quicktest v1.14.0/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09mUdL/ijj8og= +github.com/frankban/quicktest v1.14.2 h1:SPb1KFFmM+ybpEjPUhCCkZOM5xlovT5UbrMvWnXyBns= +github.com/frankban/quicktest v1.14.2/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI= @@ -329,8 +330,9 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o= +github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -623,12 +625,13 @@ github.com/ipld/go-ipld-prime v0.12.3/go.mod h1:PaeLYq8k6dJLmDUSLrzkEpoGV4PEfe/1 github.com/ipld/go-ipld-prime v0.14.0/go.mod h1:9ASQLwUFLptCov6lIYc70GRB4V7UTyLD0IJtrDJe6ZM= github.com/ipld/go-ipld-prime v0.14.1/go.mod h1:QcE4Y9n/ZZr8Ijg5bGPT0GqYWgZ1704nH0RDcQtgTP0= github.com/ipld/go-ipld-prime v0.14.2/go.mod h1:QcE4Y9n/ZZr8Ijg5bGPT0GqYWgZ1704nH0RDcQtgTP0= -github.com/ipld/go-ipld-prime v0.14.4 h1:bqhmume8+nbNsX4/+J6eohktfZHAI8GKrF3rQ0xgOyc= github.com/ipld/go-ipld-prime v0.14.4/go.mod h1:QcE4Y9n/ZZr8Ijg5bGPT0GqYWgZ1704nH0RDcQtgTP0= +github.com/ipld/go-ipld-prime v0.16.0 h1:RS5hhjB/mcpeEPJvfyj0qbOj/QL+/j05heZ0qa97dVo= +github.com/ipld/go-ipld-prime v0.16.0/go.mod h1:axSCuOCBPqrH+gvXr2w9uAOulJqBPhHPT2PjoiiU1qA= github.com/ipld/go-ipld-prime-proto v0.0.0-20191113031812-e32bd156a1e5/go.mod h1:gcvzoEDBjwycpXt3LBE061wT9f46szXGHAmj9uoP6fU= github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20211210234204-ce2a1c70cd73 h1:TsyATB2ZRRQGTwafJdgEUQkmjOExRV0DNokcihZxbnQ= github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20211210234204-ce2a1c70cd73/go.mod h1:2PJ0JgxyB08t0b2WKrcuqI3di0V+5n6RS/LTUJhkoxY= -github.com/ipld/go-storethehash v0.0.1/go.mod h1:w8cQfWInks8lvvbQTiKbCPusU9v0sqiViBihTHbavpQ= +github.com/ipld/go-storethehash v0.0.2/go.mod h1:w8cQfWInks8lvvbQTiKbCPusU9v0sqiViBihTHbavpQ= github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52 h1:QG4CGBqCeuBo6aZlGAamSkxWdgWfZGeE49eUOWJPA4c= github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52/go.mod h1:fdg+/X9Gg4AsAIzWpEHwnqd+QY3b7lajxyjE1m4hkq4= github.com/jackpal/gateway v1.0.5/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA= @@ -1447,7 +1450,10 @@ github.com/whyrusleeping/cbor-gen v0.0.0-20200810223238-211df3b9e24c/go.mod h1:f github.com/whyrusleeping/cbor-gen v0.0.0-20200812213548-958ddffe352c/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= github.com/whyrusleeping/cbor-gen v0.0.0-20200826160007-0b9f6c5fb163/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= github.com/whyrusleeping/cbor-gen v0.0.0-20210219115102-f37d292932f2/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= +<<<<<<< HEAD github.com/whyrusleeping/cbor-gen v0.0.0-20220224212727-7a699437a831/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= +======= +>>>>>>> Use new metadata api with multi-protocol support github.com/whyrusleeping/cbor-gen v0.0.0-20220302191723-37c43cae8e14 h1:vo2wkP2ceHyGyZwFFtAabpot03EeSxxwAe57pOI9E/4= github.com/whyrusleeping/cbor-gen v0.0.0-20220302191723-37c43cae8e14/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f h1:jQa4QT2UP9WYv2nzyawpKMOCl+Z/jW7djv2/J50lj9E= diff --git a/interface.go b/interface.go index acd67254..d65aea61 100644 --- a/interface.go +++ b/interface.go @@ -56,7 +56,7 @@ type Interface interface { // NotifyPut, then ErrAlreadyAdvertised is returned. // // This function returns the ID of the advertisement published. - NotifyPut(ctx context.Context, contextID []byte, metadata stiapi.Metadata) (cid.Cid, error) + NotifyPut(ctx context.Context, contextID []byte, metadata stiapi.ParsedMetadata) (cid.Cid, error) // NotifyRemove signals to the provider that the multihashes that // corresponded to the given contextID are no longer available. An advertisement diff --git a/metadata/bitswap.go b/metadata/bitswap.go index b304f015..db705bff 100644 --- a/metadata/bitswap.go +++ b/metadata/bitswap.go @@ -1,11 +1,49 @@ package metadata import ( + "errors" + stiapi "github.com/filecoin-project/storetheindex/api/v0" "github.com/multiformats/go-multicodec" + "github.com/multiformats/go-varint" ) // BitswapMetadata represents the indexing metadata that uses multicodec.TransportBitswap. -var BitswapMetadata = stiapi.Metadata{ - ProtocolID: multicodec.TransportBitswap, +type BitswapMetadata struct { +} + +func init() { + stiapi.RegisterMetadataProtocol(func() stiapi.ProtocolMetadata { + return &BitswapMetadata{} + }) +} + +func (m *BitswapMetadata) Protocol() multicodec.Code { + return multicodec.TransportBitswap +} + +func (m *BitswapMetadata) PayloadLength() int { + // No data, so return size of Uvarint. + return varint.UvarintSize(0) +} + +func (m *BitswapMetadata) MarshalBinary() ([]byte, error) { + buf := make([]byte, varint.UvarintSize(0)) + varint.PutUvarint(buf, 0) + return buf, nil +} + +func (m *BitswapMetadata) UnmarshalBinary(data []byte) error { + l, _, err := varint.FromUvarint(data) + if err != nil { + return err + } + if l != 0 { + return errors.New("bitswap data lenght not zero") + } + return nil +} + +func (m *BitswapMetadata) Equal(other stiapi.ProtocolMetadata) bool { + return other.Protocol() == m.Protocol() } diff --git a/metadata/graphsync_filecoinv1.go b/metadata/graphsync_filecoinv1.go index 7b4cb7ce..acd61403 100644 --- a/metadata/graphsync_filecoinv1.go +++ b/metadata/graphsync_filecoinv1.go @@ -2,6 +2,7 @@ package metadata import ( "bytes" + "errors" "fmt" stiapi "github.com/filecoin-project/storetheindex/api/v0" @@ -10,6 +11,7 @@ import ( "github.com/ipld/go-ipld-prime/node/bindnode" "github.com/ipld/go-ipld-prime/schema" "github.com/multiformats/go-multicodec" + "github.com/multiformats/go-varint" ) // GraphsyncFilecoinV1Metadata represents the indexing metadata for multicodec.TransportGraphsyncFilecoinv1. @@ -22,6 +24,78 @@ type GraphsyncFilecoinV1Metadata struct { FastRetrieval bool } +var ParsedGraphsyncFilecoinV1MetadataMetadata = stiapi.ParsedMetadata{Protocols: []stiapi.ProtocolMetadata{&GraphsyncFilecoinV1Metadata{}}} + +func init() { + stiapi.RegisterMetadataProtocol(func() stiapi.ProtocolMetadata { + return &GraphsyncFilecoinV1Metadata{} + }) +} + +func (m *GraphsyncFilecoinV1Metadata) Protocol() multicodec.Code { + return multicodec.TransportGraphsyncFilecoinv1 +} + +func (dtm *GraphsyncFilecoinV1Metadata) PayloadLength() int { + nd := bindnode.Wrap(dtm, graphSyncfilecoinV1SchemaType) + encLen, err := dagcbor.EncodedLength(nd) + if err != nil { + panic(err) + } + return varint.UvarintSize(uint64(encLen)) + int(encLen) +} + +func (dtm *GraphsyncFilecoinV1Metadata) MarshalBinary() ([]byte, error) { + nd := bindnode.Wrap(dtm, graphSyncfilecoinV1SchemaType) + buf := new(bytes.Buffer) + err := dagcbor.Encode(nd, buf) + if err != nil { + return stiapi.Metadata{}, err + } + + uviBuf := varint.ToUvarint(uint64(buf.Len())) + + var b bytes.Buffer + b.Grow(len(uviBuf) + buf.Len()) + b.Write(uviBuf) + b.Write(buf.Bytes()) + + return b.Bytes(), nil +} + +func (dtm *GraphsyncFilecoinV1Metadata) UnmarshalBinary(data []byte) error { + l, sl, err := varint.FromUvarint(data) + if err != nil { + return err + } + if int(l)+sl < len(data) { + return errors.New("payload too short") + } + data = data[sl:] + + r := bytes.NewBuffer(data) + proto := bindnode.Prototype((*GraphsyncFilecoinV1Metadata)(nil), graphSyncfilecoinV1SchemaType) + nb := proto.NewBuilder() + err = dagcbor.Decode(nb, r) + if err != nil { + return err + } + nd := nb.Build() + gm := bindnode.Unwrap(nd).(*GraphsyncFilecoinV1Metadata) + dtm.VerifiedDeal = gm.VerifiedDeal + dtm.FastRetrieval = gm.FastRetrieval + dtm.PieceCID = gm.PieceCID + return nil +} + +func (m GraphsyncFilecoinV1Metadata) Equal(other stiapi.ProtocolMetadata) bool { + if other.Protocol() != m.Protocol() { + return false + } + ogfm := other.(*GraphsyncFilecoinV1Metadata) + return ogfm.PieceCID == m.PieceCID && ogfm.VerifiedDeal == m.VerifiedDeal && ogfm.FastRetrieval == m.FastRetrieval +} + var graphSyncfilecoinV1SchemaType schema.Type func init() { @@ -41,38 +115,27 @@ func init() { } // ToIndexerMetadata converts GraphsyncFilecoinV1Metadata into indexer Metadata format. -func (dtm *GraphsyncFilecoinV1Metadata) ToIndexerMetadata() (stiapi.Metadata, error) { - nd := bindnode.Wrap(dtm, graphSyncfilecoinV1SchemaType) - buf := new(bytes.Buffer) - err := dagcbor.Encode(nd, buf) - if err != nil { - return stiapi.Metadata{}, err - } - return stiapi.Metadata{ - ProtocolID: multicodec.TransportGraphsyncFilecoinv1, - Data: buf.Bytes(), - }, nil +func (dtm *GraphsyncFilecoinV1Metadata) ToIndexerMetadata() (stiapi.ParsedMetadata, error) { + return stiapi.ParsedMetadata{Protocols: []stiapi.ProtocolMetadata{dtm}}, nil } // FromIndexerMetadata decodes the indexer metadata format into GraphsyncFilecoinV1Metadata // if the protocol ID of the given metadata matches multicodec.TransportGraphsyncFilecoinv1. // Otherwise, ErrNotGraphsyncFilecoinV1 error is returned. -func (dtm *GraphsyncFilecoinV1Metadata) FromIndexerMetadata(m stiapi.Metadata) error { - if m.ProtocolID != multicodec.TransportGraphsyncFilecoinv1 { - return ErrNotGraphsyncFilecoinV1{m.ProtocolID} +func (dtm *GraphsyncFilecoinV1Metadata) FromIndexerMetadata(m stiapi.ParsedMetadata) error { + var pm stiapi.ProtocolMetadata + var incorrectProtocol multicodec.Code + for _, p := range m.Protocols { + if p.Protocol() == multicodec.TransportGraphsyncFilecoinv1 { + pm = p + break + } + incorrectProtocol = p.Protocol() } - r := bytes.NewBuffer(m.Data) - proto := bindnode.Prototype((*GraphsyncFilecoinV1Metadata)(nil), graphSyncfilecoinV1SchemaType) - nb := proto.NewBuilder() - err := dagcbor.Decode(nb, r) - if err != nil { - return err + if pm == nil { + return ErrNotGraphsyncFilecoinV1{incorrectProtocol} } - nd := nb.Build() - gm := bindnode.Unwrap(nd).(*GraphsyncFilecoinV1Metadata) - dtm.VerifiedDeal = gm.VerifiedDeal - dtm.FastRetrieval = gm.FastRetrieval - dtm.PieceCID = gm.PieceCID + *dtm = *(pm.(*GraphsyncFilecoinV1Metadata)) return nil } diff --git a/metadata/graphsync_filecoinv1_test.go b/metadata/graphsync_filecoinv1_test.go index fb6e93e4..0a069ea0 100644 --- a/metadata/graphsync_filecoinv1_test.go +++ b/metadata/graphsync_filecoinv1_test.go @@ -1,11 +1,12 @@ package metadata_test import ( - stiapi "github.com/filecoin-project/storetheindex/api/v0" "testing" "github.com/filecoin-project/index-provider/metadata" "github.com/filecoin-project/index-provider/testutil" + stiapi "github.com/filecoin-project/storetheindex/api/v0" + v0util "github.com/filecoin-project/storetheindex/api/v0/util" "github.com/multiformats/go-multicodec" "github.com/stretchr/testify/require" ) @@ -37,7 +38,8 @@ func TestRoundTripDataTransferFilecoin(t *testing.T) { for _, src := range filecoinV1Datas { imd, err := src.ToIndexerMetadata() require.NoError(t, err) - require.Equal(t, multicodec.TransportGraphsyncFilecoinv1, imd.ProtocolID) + require.Equal(t, 1, len(imd.Protocols)) + require.Equal(t, multicodec.TransportGraphsyncFilecoinv1, imd.Protocols[0].Protocol()) dst := &metadata.GraphsyncFilecoinV1Metadata{} err = dst.FromIndexerMetadata(imd) @@ -47,11 +49,9 @@ func TestRoundTripDataTransferFilecoin(t *testing.T) { } func TestGraphsyncFilecoinV1Metadata_FromIndexerMetadataErr(t *testing.T) { - imd := stiapi.Metadata{ - ProtocolID: multicodec.TransportBitswap, - } + imd := stiapi.ParsedMetadata{Protocols: []stiapi.ProtocolMetadata{&v0util.ExampleMetadata{Data: []byte("metadata")}}} dst := &metadata.GraphsyncFilecoinV1Metadata{} err := dst.FromIndexerMetadata(imd) require.IsType(t, metadata.ErrNotGraphsyncFilecoinV1{}, err) - require.Equal(t, "protocol 0x900 does not use the FilecoinV1 exchange format", err.Error()) + require.Equal(t, "protocol 0x300000 does not use the FilecoinV1 exchange format", err.Error()) } diff --git a/mock/interface.go b/mock/interface.go index 7a2d779a..5514342c 100644 --- a/mock/interface.go +++ b/mock/interface.go @@ -71,7 +71,7 @@ func (mr *MockInterfaceMockRecorder) GetLatestAdv(arg0 interface{}) *gomock.Call } // NotifyPut mocks base method. -func (m *MockInterface) NotifyPut(ctx context.Context, contextID []byte, metadata v0.Metadata) (cid.Cid, error) { +func (m *MockInterface) NotifyPut(ctx context.Context, contextID []byte, metadata v0.ParsedMetadata) (cid.Cid, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "NotifyPut", ctx, contextID, metadata) ret0, _ := ret[0].(cid.Cid) diff --git a/server/admin/http/importcar_handler.go b/server/admin/http/importcar_handler.go index 80096679..228a7c3e 100644 --- a/server/admin/http/importcar_handler.go +++ b/server/admin/http/importcar_handler.go @@ -7,6 +7,7 @@ import ( "github.com/filecoin-project/index-provider" "github.com/filecoin-project/index-provider/supplier" + stiapi "github.com/filecoin-project/storetheindex/api/v0" "github.com/ipfs/go-cid" ) @@ -32,7 +33,14 @@ func (h *importCarHandler) handle(w http.ResponseWriter, r *http.Request) { ctx := context.Background() log.Info("importing CAR") - advID, err = h.cs.Put(ctx, req.Key, req.Path, req.Metadata) + var parsedMetadata stiapi.ParsedMetadata + if err = parsedMetadata.UnmarshalBinary(req.Metadata); err != nil { + msg := fmt.Sprintf("failed to unmarshal binary: %v", err) + log.Errorw(msg, "err", err) + http.Error(w, msg, http.StatusBadRequest) + return + } + advID, err = h.cs.Put(ctx, req.Key, req.Path, parsedMetadata) // Respond with cause of failure. if err != nil { diff --git a/server/admin/http/importcar_handler_test.go b/server/admin/http/importcar_handler_test.go index 84f21dc2..84cb992e 100644 --- a/server/admin/http/importcar_handler_test.go +++ b/server/admin/http/importcar_handler_test.go @@ -27,10 +27,13 @@ func Test_importCarHandler(t *testing.T) { wantKey := []byte("lobster") wantMetadata, err := cardatatransfer.MetadataFromContextID(wantKey) require.NoError(t, err) + wantRawMetadata, err := wantMetadata.MarshalBinary() + require.NoError(t, err) + icReq := &ImportCarReq{ Path: "fish", Key: wantKey, - Metadata: wantMetadata, + Metadata: wantRawMetadata, } jsonReq, err := json.Marshal(icReq) @@ -76,10 +79,13 @@ func Test_importCarHandlerFail(t *testing.T) { wantKey := []byte("lobster") wantMetadata, err := cardatatransfer.MetadataFromContextID(wantKey) require.NoError(t, err) + wantRawMetadata, err := wantMetadata.MarshalBinary() + require.NoError(t, err) + icReq := &ImportCarReq{ Path: "fish", Key: wantKey, - Metadata: wantMetadata, + Metadata: wantRawMetadata, } jsonReq, err := json.Marshal(icReq) @@ -117,10 +123,13 @@ func Test_importCarAlreadyAdvertised(t *testing.T) { wantKey := []byte("lobster") wantMetadata, err := cardatatransfer.MetadataFromContextID(wantKey) require.NoError(t, err) + wantRawMetadata, err := wantMetadata.MarshalBinary() + require.NoError(t, err) + icReq := &ImportCarReq{ Path: "fish", Key: wantKey, - Metadata: wantMetadata, + Metadata: wantRawMetadata, } jsonReq, err := json.Marshal(icReq) diff --git a/supplier/car_supplier.go b/supplier/car_supplier.go index 024840f8..1b4812ef 100644 --- a/supplier/car_supplier.go +++ b/supplier/car_supplier.go @@ -59,7 +59,7 @@ func NewCarSupplier(eng provider.Interface, ds datastore.Datastore, opts ...car. // get an iterator over CIDs that belong to the CAR. // // This function accepts both CARv1 and CARv2 formats. -func (cs *CarSupplier) Put(ctx context.Context, contextID []byte, path string, metadata stiapi.Metadata) (cid.Cid, error) { +func (cs *CarSupplier) Put(ctx context.Context, contextID []byte, path string, metadata stiapi.ParsedMetadata) (cid.Cid, error) { // Clean path to CAR. path = filepath.Clean(path) diff --git a/supplier/car_supplier_test.go b/supplier/car_supplier_test.go index fd903933..f09e9531 100644 --- a/supplier/car_supplier_test.go +++ b/supplier/car_supplier_test.go @@ -9,6 +9,7 @@ import ( "os" "testing" + "github.com/filecoin-project/index-provider/metadata" mock_provider "github.com/filecoin-project/index-provider/mock" stiapi "github.com/filecoin-project/storetheindex/api/v0" "github.com/golang/mock/gomock" @@ -19,8 +20,6 @@ import ( "github.com/stretchr/testify/require" ) -const testProtocolID = 0x300000 - func TestPutCarReturnsExpectedIterator(t *testing.T) { rng := rand.New(rand.NewSource(1413)) tests := []struct { @@ -47,9 +46,8 @@ func TestPutCarReturnsExpectedIterator(t *testing.T) { carPath: "../testdata/sample-wrapped-v2.car", }, } - metadata := stiapi.Metadata{ - ProtocolID: testProtocolID, - } + parsedMetadata := stiapi.ParsedMetadata{Protocols: []stiapi.ProtocolMetadata{&metadata.GraphsyncFilecoinV1Metadata{}}} + for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { mc := gomock.NewController(t) @@ -94,11 +92,11 @@ func TestPutCarReturnsExpectedIterator(t *testing.T) { wantCid := generateCidV1(t, rng) mockEng. EXPECT(). - NotifyPut(ctx, gomock.Any(), metadata). + NotifyPut(ctx, gomock.Any(), parsedMetadata). Return(wantCid, nil) gotContextID := sha256.New().Sum([]byte(tt.carPath)) - gotCid, err := subject.Put(ctx, gotContextID, tt.carPath, metadata) + gotCid, err := subject.Put(ctx, gotContextID, tt.carPath, parsedMetadata) require.NoError(t, err) require.Equal(t, wantCid, gotCid) @@ -143,18 +141,16 @@ func TestRemovedPathIsNoLongerSupplied(t *testing.T) { subject := NewCarSupplier(mockEng, ds) t.Cleanup(func() { require.NoError(t, subject.Close()) }) - metadata := stiapi.Metadata{ - ProtocolID: testProtocolID, - } + parsedMetadata := stiapi.ParsedMetadata{Protocols: []stiapi.ProtocolMetadata{&metadata.GraphsyncFilecoinV1Metadata{}}} wantCid := generateCidV1(t, rng) mockEng. EXPECT(). - NotifyPut(ctx, gomock.Any(), metadata). + NotifyPut(ctx, gomock.Any(), parsedMetadata). Return(wantCid, nil) gotContextID := sha256.New().Sum([]byte(path)) - id, err := subject.Put(ctx, gotContextID, path, metadata) + id, err := subject.Put(ctx, gotContextID, path, parsedMetadata) require.NoError(t, err) require.Equal(t, wantCid, id) From cd15047da366adb6726c8cd87419e2ccf98aa127 Mon Sep 17 00:00:00 2001 From: gammazero Date: Tue, 15 Mar 2022 03:40:29 -0700 Subject: [PATCH 2/3] Commands use new metadata api --- cmd/provider/find.go | 2 +- cmd/provider/import.go | 18 ++++++++++++------ cmd/provider/index.go | 11 ++++++----- 3 files changed, 19 insertions(+), 12 deletions(-) diff --git a/cmd/provider/find.go b/cmd/provider/find.go index 97df3bb0..887d9633 100644 --- a/cmd/provider/find.go +++ b/cmd/provider/find.go @@ -86,7 +86,7 @@ func findCommand(cctx *cli.Context) error { for _, pr := range resp.MultihashResults[i].ProviderResults { fmt.Println(" Provider:", pr.Provider) fmt.Println(" ContextID:", base64.StdEncoding.EncodeToString(pr.ContextID)) - fmt.Println(" Metadata:", base64.StdEncoding.EncodeToString(pr.Metadata.Data)) + fmt.Println(" Metadata:", base64.StdEncoding.EncodeToString(*pr.Metadata)) } } diff --git a/cmd/provider/import.go b/cmd/provider/import.go index 11fa6989..70a6d067 100644 --- a/cmd/provider/import.go +++ b/cmd/provider/import.go @@ -11,7 +11,6 @@ import ( "github.com/filecoin-project/index-provider/cardatatransfer" adminserver "github.com/filecoin-project/index-provider/server/admin/http" stiapi "github.com/filecoin-project/storetheindex/api/v0" - "github.com/multiformats/go-multicodec" "github.com/urfave/cli/v2" ) @@ -32,9 +31,8 @@ var ( Before: beforeImportCar, Action: doImportCar, } - metadata = stiapi.Metadata{ - ProtocolID: multicodec.TransportGraphsyncFilecoinv1, - } + + metadata stiapi.ParsedMetadata ) func beforeImportCar(cctx *cli.Context) error { @@ -54,7 +52,10 @@ func beforeImportCar(cctx *cli.Context) error { if err != nil { return errors.New("metadata is not a valid base64 encoded string") } - metadata.Data = decoded + err = metadata.UnmarshalBinary(decoded) + if err != nil { + return err + } } else { // If no metadata is set, generate metadata that is compatible for FileCoin retrieval base // on the context ID @@ -68,10 +69,15 @@ func beforeImportCar(cctx *cli.Context) error { } func doImportCar(cctx *cli.Context) error { + rawMetadata, err := metadata.MarshalBinary() + if err != nil { + return err + } + req := adminserver.ImportCarReq{ Path: carPathFlagValue, Key: importCarKey, - Metadata: metadata, + Metadata: rawMetadata, } resp, err := doHttpPostReq(cctx.Context, adminAPIFlagValue+"/admin/import/car", req) if err != nil { diff --git a/cmd/provider/index.go b/cmd/provider/index.go index 9445578b..a6bab3ff 100644 --- a/cmd/provider/index.go +++ b/cmd/provider/index.go @@ -5,10 +5,10 @@ import ( "fmt" "github.com/filecoin-project/index-provider/cmd/provider/internal/config" + md "github.com/filecoin-project/index-provider/metadata" stiapi "github.com/filecoin-project/storetheindex/api/v0" httpc "github.com/filecoin-project/storetheindex/api/v0/ingest/client/http" "github.com/ipfs/go-cid" - "github.com/multiformats/go-multicodec" "github.com/multiformats/go-multihash" "github.com/urfave/cli/v2" ) @@ -61,12 +61,13 @@ func indexCommand(cctx *cli.Context) error { return err } - metadata := stiapi.Metadata{ - ProtocolID: multicodec.Code(cctx.Int("proto")), - Data: []byte(cctx.String("meta")), + metadata := stiapi.ParsedMetadata{Protocols: []stiapi.ProtocolMetadata{&md.GraphsyncFilecoinV1Metadata{}}} + rawMetadata, err := metadata.MarshalBinary() + if err != nil { + return err } - err = client.IndexContent(cctx.Context, peerID, privKey, mh, []byte(cctx.String("ctxid")), metadata, cctx.StringSlice("addr")) + err = client.IndexContent(cctx.Context, peerID, privKey, mh, []byte(cctx.String("ctxid")), rawMetadata, cctx.StringSlice("addr")) if err != nil { return err } From 01cc67975fcc8d1a11d2d21c09d12f41b55c9aa5 Mon Sep 17 00:00:00 2001 From: gammazero Date: Tue, 15 Mar 2022 10:15:02 -0700 Subject: [PATCH 3/3] Rebase and fix conflicts --- cmd/go.sum | 4 ---- engine/engine_test.go | 22 +++++++++++++++++----- go.sum | 4 ---- 3 files changed, 17 insertions(+), 13 deletions(-) diff --git a/cmd/go.sum b/cmd/go.sum index d5dd3028..0a88da84 100644 --- a/cmd/go.sum +++ b/cmd/go.sum @@ -1465,10 +1465,6 @@ github.com/whyrusleeping/cbor-gen v0.0.0-20200810223238-211df3b9e24c/go.mod h1:f github.com/whyrusleeping/cbor-gen v0.0.0-20200812213548-958ddffe352c/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= github.com/whyrusleeping/cbor-gen v0.0.0-20200826160007-0b9f6c5fb163/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= github.com/whyrusleeping/cbor-gen v0.0.0-20210219115102-f37d292932f2/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= -<<<<<<< HEAD -github.com/whyrusleeping/cbor-gen v0.0.0-20220224212727-7a699437a831/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= -======= ->>>>>>> Use new metadata api with multi-protocol support github.com/whyrusleeping/cbor-gen v0.0.0-20220302191723-37c43cae8e14 h1:vo2wkP2ceHyGyZwFFtAabpot03EeSxxwAe57pOI9E/4= github.com/whyrusleeping/cbor-gen v0.0.0-20220302191723-37c43cae8e14/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f h1:jQa4QT2UP9WYv2nzyawpKMOCl+Z/jW7djv2/J50lj9E= diff --git a/engine/engine_test.go b/engine/engine_test.go index a7c18fec..6c640d04 100644 --- a/engine/engine_test.go +++ b/engine/engine_test.go @@ -15,8 +15,8 @@ import ( "github.com/filecoin-project/index-provider/engine" "github.com/filecoin-project/index-provider/metadata" "github.com/filecoin-project/index-provider/testutil" + stiapi "github.com/filecoin-project/storetheindex/api/v0" "github.com/filecoin-project/storetheindex/api/v0/ingest/schema" - v0util "github.com/filecoin-project/storetheindex/api/v0/util" "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore" dssync "github.com/ipfs/go-datastore/sync" @@ -59,12 +59,16 @@ func TestEngine_PublishLocal(t *testing.T) { }) require.NoError(t, err) + metadata := stiapi.ParsedMetadata{Protocols: []stiapi.ProtocolMetadata{&metadata.BitswapMetadata{}}} + encMeta, err := metadata.MarshalBinary() + require.NoError(t, err) + wantAd, err := schema.NewAdvertisement( subject.Key(), nil, chunkLnk, []byte("fish"), - metadata.BitswapMetadata, + stiapi.Metadata(encMeta), false, subject.Host().ID().String(), multiAddsToString(subject.Host().Addrs())) @@ -150,12 +154,16 @@ func TestEngine_PublishWithDataTransferPublisher(t *testing.T) { }) require.NoError(t, err) + metadata := stiapi.ParsedMetadata{Protocols: []stiapi.ProtocolMetadata{&metadata.BitswapMetadata{}}} + encMeta, err := metadata.MarshalBinary() + require.NoError(t, err) + wantAd, err := schema.NewAdvertisement( subject.Key(), nil, chunkLnk, wantContextID, - metadata.BitswapMetadata, + stiapi.Metadata(encMeta), false, subject.Host().ID().String(), multiAddsToString(subject.Host().Addrs())) @@ -225,7 +233,9 @@ func TestEngine_NotifyPutWithoutListerIsError(t *testing.T) { require.NoError(t, err) defer subject.Shutdown() - gotCid, err := subject.NotifyPut(ctx, []byte("fish"), metadata.BitswapMetadata) + parsedMetadata := stiapi.ParsedMetadata{Protocols: []stiapi.ProtocolMetadata{&metadata.BitswapMetadata{}}} + + gotCid, err := subject.NotifyPut(ctx, []byte("fish"), parsedMetadata) require.Error(t, err, provider.ErrNoMultihashLister) require.Equal(t, cid.Undef, gotCid) } @@ -253,7 +263,9 @@ func TestEngine_NotifyPutThenNotifyRemove(t *testing.T) { return nil, errors.New("not found") }) - gotPutAdCid, err := subject.NotifyPut(ctx, wantContextID, metadata.BitswapMetadata) + parsedMetadata := stiapi.ParsedMetadata{Protocols: []stiapi.ProtocolMetadata{&metadata.BitswapMetadata{}}} + + gotPutAdCid, err := subject.NotifyPut(ctx, wantContextID, parsedMetadata) require.NoError(t, err) require.NotEqual(t, cid.Undef, gotPutAdCid) diff --git a/go.sum b/go.sum index 9a5135f6..34513630 100644 --- a/go.sum +++ b/go.sum @@ -1450,10 +1450,6 @@ github.com/whyrusleeping/cbor-gen v0.0.0-20200810223238-211df3b9e24c/go.mod h1:f github.com/whyrusleeping/cbor-gen v0.0.0-20200812213548-958ddffe352c/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= github.com/whyrusleeping/cbor-gen v0.0.0-20200826160007-0b9f6c5fb163/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= github.com/whyrusleeping/cbor-gen v0.0.0-20210219115102-f37d292932f2/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= -<<<<<<< HEAD -github.com/whyrusleeping/cbor-gen v0.0.0-20220224212727-7a699437a831/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= -======= ->>>>>>> Use new metadata api with multi-protocol support github.com/whyrusleeping/cbor-gen v0.0.0-20220302191723-37c43cae8e14 h1:vo2wkP2ceHyGyZwFFtAabpot03EeSxxwAe57pOI9E/4= github.com/whyrusleeping/cbor-gen v0.0.0-20220302191723-37c43cae8e14/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f h1:jQa4QT2UP9WYv2nzyawpKMOCl+Z/jW7djv2/J50lj9E=