From 1dd06637d3d3dc834b5ff35fa50f5ab6063042cd Mon Sep 17 00:00:00 2001 From: Vicente Olmedo Date: Mon, 16 Dec 2024 17:54:33 +0100 Subject: [PATCH] fix: fetch from legacy if there are no interesting claims in IPNI (#61) With the current implementation, we only look in legacy claims storage if IPNI returns no results. This PR changes this so that we also search in legacy claims when IPNI return results that are not interesting for the current query. --- pkg/internal/testutil/gen.go | 24 ++++++++ pkg/service/providerindex/providerindex.go | 60 +++++++++++++------ .../providerindex/providerindex_test.go | 59 ++++++++++++++---- 3 files changed, 113 insertions(+), 30 deletions(-) diff --git a/pkg/internal/testutil/gen.go b/pkg/internal/testutil/gen.go index 32cd248..dd00b86 100644 --- a/pkg/internal/testutil/gen.go +++ b/pkg/internal/testutil/gen.go @@ -13,12 +13,14 @@ import ( "github.com/ipld/go-ipld-prime/datamodel" cidlink "github.com/ipld/go-ipld-prime/linking/cid" "github.com/ipni/go-libipni/find/model" + ipnimeta "github.com/ipni/go-libipni/metadata" crypto "github.com/libp2p/go-libp2p/core/crypto" peer "github.com/libp2p/go-libp2p/core/peer" "github.com/multiformats/go-multiaddr" manet "github.com/multiformats/go-multiaddr/net" mh "github.com/multiformats/go-multihash" "github.com/storacha/go-capabilities/pkg/assert" + "github.com/storacha/go-metadata" "github.com/storacha/go-ucanto/core/car" "github.com/storacha/go-ucanto/core/delegation" "github.com/storacha/go-ucanto/core/ipld/block" @@ -172,6 +174,28 @@ func RandomProviderResult() model.ProviderResult { } } +func RandomBitswapProviderResult() model.ProviderResult { + pr := RandomProviderResult() + bitswapMeta, _ := ipnimeta.Bitswap{}.MarshalBinary() + pr.Metadata = bitswapMeta + return pr +} + +func RandomLocationCommitmentProviderResult() model.ProviderResult { + shard := RandomCID().(cidlink.Link).Cid + locationMeta := metadata.LocationCommitmentMetadata{ + Shard: &shard, + Range: &metadata.Range{Offset: 128}, + Expiration: 0, + Claim: RandomCID().(cidlink.Link).Cid, + } + metaBytes, _ := locationMeta.MarshalBinary() + + pr := RandomProviderResult() + pr.Metadata = metaBytes + return pr +} + func RandomShardedDagIndexView(size int) (mh.Multihash, blobindex.ShardedDagIndexView) { root, digest, bytes := RandomCAR(size) shard, err := blobindex.FromShardArchives(root, [][]byte{bytes}) diff --git a/pkg/service/providerindex/providerindex.go b/pkg/service/providerindex/providerindex.go index e712991..838cba0 100644 --- a/pkg/service/providerindex/providerindex.go +++ b/pkg/service/providerindex/providerindex.go @@ -58,18 +58,15 @@ func New(providerStore types.ProviderStore, findClient ipnifind.Finder, publishe // encodedcontextid's by hashing space DID and Hash, and filter for a matching context id // Future TODO: kick off a conversion task to update the records func (pi *ProviderIndexService) Find(ctx context.Context, qk QueryKey) ([]model.ProviderResult, error) { - results, err := pi.getProviderResults(ctx, qk.Hash) + results, err := pi.getProviderResults(ctx, qk.Hash, qk.TargetClaims) if err != nil { return nil, err } - results, err = pi.filteredCodecs(results, qk.TargetClaims) - if err != nil { - return nil, err - } - return pi.filterBySpace(results, qk.Hash, qk.Spaces) + + return filterBySpace(results, qk.Hash, qk.Spaces) } -func (pi *ProviderIndexService) getProviderResults(ctx context.Context, mh mh.Multihash) ([]model.ProviderResult, error) { +func (pi *ProviderIndexService) getProviderResults(ctx context.Context, mh mh.Multihash, targetClaims []multicodec.Code) ([]model.ProviderResult, error) { res, err := pi.providerStore.Get(ctx, mh) if err == nil { return res, nil @@ -78,20 +75,14 @@ func (pi *ProviderIndexService) getProviderResults(ctx context.Context, mh mh.Mu return nil, err } - var results []model.ProviderResult - - findRes, err := pi.findClient.Find(ctx, mh) + results, err := pi.fetchFromIPNI(ctx, mh, targetClaims) if err != nil { return nil, err } - for _, mhres := range findRes.MultihashResults { - results = append(results, mhres.ProviderResults...) - } - - // try legacy claims storage if nothing was found on IPNI + // if nothing was found on IPNI, try legacy claims storage if len(results) == 0 { - legacyResults, err := pi.legacyClaims.Find(ctx, mh) + legacyResults, err := pi.fetchFromLegacy(ctx, mh, targetClaims) if err != nil { return nil, err } @@ -109,7 +100,40 @@ func (pi *ProviderIndexService) getProviderResults(ctx context.Context, mh mh.Mu return results, nil } -func (pi *ProviderIndexService) filteredCodecs(results []model.ProviderResult, codecs []multicodec.Code) ([]model.ProviderResult, error) { +func (pi *ProviderIndexService) fetchFromIPNI(ctx context.Context, mh mh.Multihash, targetClaims []multicodec.Code) ([]model.ProviderResult, error) { + var results []model.ProviderResult + findRes, err := pi.findClient.Find(ctx, mh) + if err != nil { + return nil, err + } + + for _, mhres := range findRes.MultihashResults { + results = append(results, mhres.ProviderResults...) + } + + results, err = filterCodecs(results, targetClaims) + if err != nil { + return nil, err + } + + return results, nil +} + +func (pi *ProviderIndexService) fetchFromLegacy(ctx context.Context, mh mh.Multihash, targetClaims []multicodec.Code) ([]model.ProviderResult, error) { + results, err := pi.legacyClaims.Find(ctx, mh) + if err != nil { + return nil, err + } + + results, err = filterCodecs(results, targetClaims) + if err != nil { + return nil, err + } + + return results, nil +} + +func filterCodecs(results []model.ProviderResult, codecs []multicodec.Code) ([]model.ProviderResult, error) { if len(codecs) == 0 { return results, nil } @@ -127,7 +151,7 @@ func (pi *ProviderIndexService) filteredCodecs(results []model.ProviderResult, c }) } -func (pi *ProviderIndexService) filterBySpace(results []model.ProviderResult, mh mh.Multihash, spaces []did.DID) ([]model.ProviderResult, error) { +func filterBySpace(results []model.ProviderResult, mh mh.Multihash, spaces []did.DID) ([]model.ProviderResult, error) { if len(spaces) == 0 { return results, nil } diff --git a/pkg/service/providerindex/providerindex_test.go b/pkg/service/providerindex/providerindex_test.go index cc00bc0..edc8ac6 100644 --- a/pkg/service/providerindex/providerindex_test.go +++ b/pkg/service/providerindex/providerindex_test.go @@ -6,6 +6,8 @@ import ( "testing" "github.com/ipni/go-libipni/find/model" + "github.com/multiformats/go-multicodec" + "github.com/storacha/go-metadata" "github.com/storacha/indexing-service/pkg/internal/testutil" "github.com/storacha/indexing-service/pkg/internal/testutil/mocks" "github.com/storacha/indexing-service/pkg/types" @@ -28,7 +30,7 @@ func TestGetProviderResults(t *testing.T) { mockStore.EXPECT().Get(ctx, someHash).Return([]model.ProviderResult{expectedResult}, nil) - results, err := providerIndex.getProviderResults(ctx, someHash) + results, err := providerIndex.getProviderResults(ctx, someHash, []multicodec.Code{0}) require.NoError(t, err) require.Equal(t, []model.ProviderResult{expectedResult}, results) @@ -43,7 +45,7 @@ func TestGetProviderResults(t *testing.T) { providerIndex := New(mockStore, mockIpniFinder, mockIpniPublisher, mockLegacyClaims) someHash := testutil.RandomMultihash() - expectedResult := testutil.RandomProviderResult() + expectedResult := testutil.RandomLocationCommitmentProviderResult() ipniFinderResponse := &model.FindResponse{ MultihashResults: []model.MultihashResult{ { @@ -59,13 +61,13 @@ func TestGetProviderResults(t *testing.T) { mockIpniFinder.EXPECT().Find(ctx, someHash).Return(ipniFinderResponse, nil) mockStore.EXPECT().Set(ctx, someHash, []model.ProviderResult{expectedResult}, true).Return(nil) - results, err := providerIndex.getProviderResults(ctx, someHash) + results, err := providerIndex.getProviderResults(ctx, someHash, []multicodec.Code{metadata.LocationCommitmentID}) require.NoError(t, err) require.Equal(t, []model.ProviderResult{expectedResult}, results) }) - t.Run("results not cached, not found in IPNI, found in legacy claims service, results cached afterwards", func(t *testing.T) { + t.Run("results not cached, no results from IPNI, found in legacy claims service, results cached afterwards", func(t *testing.T) { mockStore := mocks.NewMockProviderStore(t) mockIpniFinder := mocks.NewMockFinder(t) mockIpniPublisher := mocks.NewMockPublisher(t) @@ -74,7 +76,7 @@ func TestGetProviderResults(t *testing.T) { providerIndex := New(mockStore, mockIpniFinder, mockIpniPublisher, mockLegacyClaims) someHash := testutil.RandomMultihash() - expectedResult := testutil.RandomProviderResult() + expectedResult := testutil.RandomLocationCommitmentProviderResult() ctx := context.Background() @@ -83,7 +85,40 @@ func TestGetProviderResults(t *testing.T) { mockLegacyClaims.EXPECT().Find(ctx, someHash).Return([]model.ProviderResult{expectedResult}, nil) mockStore.EXPECT().Set(ctx, someHash, []model.ProviderResult{expectedResult}, true).Return(nil) - results, err := providerIndex.getProviderResults(ctx, someHash) + results, err := providerIndex.getProviderResults(ctx, someHash, []multicodec.Code{metadata.LocationCommitmentID}) + + require.NoError(t, err) + require.Equal(t, []model.ProviderResult{expectedResult}, results) + }) + + t.Run("results not cached, IPNI returns uninteresting results, search in legacy claims", func(t *testing.T) { + mockStore := mocks.NewMockProviderStore(t) + mockIpniFinder := mocks.NewMockFinder(t) + mockIpniPublisher := mocks.NewMockPublisher(t) + mockLegacyClaims := mocks.NewMockLegacyClaimsFinder(t) + + providerIndex := New(mockStore, mockIpniFinder, mockIpniPublisher, mockLegacyClaims) + + someHash := testutil.RandomMultihash() + bitswapResult := testutil.RandomBitswapProviderResult() + ipniFinderResponse := &model.FindResponse{ + MultihashResults: []model.MultihashResult{ + { + Multihash: someHash, + ProviderResults: []model.ProviderResult{bitswapResult}, + }, + }, + } + expectedResult := testutil.RandomLocationCommitmentProviderResult() + + ctx := context.Background() + + mockStore.EXPECT().Get(ctx, someHash).Return(nil, types.ErrKeyNotFound) + mockIpniFinder.EXPECT().Find(ctx, someHash).Return(ipniFinderResponse, nil) + mockLegacyClaims.EXPECT().Find(ctx, someHash).Return([]model.ProviderResult{expectedResult}, nil) + mockStore.EXPECT().Set(ctx, someHash, []model.ProviderResult{expectedResult}, true).Return(nil) + + results, err := providerIndex.getProviderResults(ctx, someHash, []multicodec.Code{metadata.LocationCommitmentID}) require.NoError(t, err) require.Equal(t, []model.ProviderResult{expectedResult}, results) @@ -105,7 +140,7 @@ func TestGetProviderResults(t *testing.T) { mockIpniFinder.EXPECT().Find(ctx, someHash).Return(&model.FindResponse{}, nil) mockLegacyClaims.EXPECT().Find(ctx, someHash).Return([]model.ProviderResult{}, nil) - results, err := providerIndex.getProviderResults(ctx, someHash) + results, err := providerIndex.getProviderResults(ctx, someHash, []multicodec.Code{0}) require.NoError(t, err) require.Empty(t, results) @@ -124,7 +159,7 @@ func TestGetProviderResults(t *testing.T) { ctx := context.Background() mockStore.EXPECT().Get(ctx, someHash).Return(nil, errors.New("some error")) - _, err := providerIndex.getProviderResults(ctx, someHash) + _, err := providerIndex.getProviderResults(ctx, someHash, []multicodec.Code{0}) require.Error(t, err) }) @@ -143,7 +178,7 @@ func TestGetProviderResults(t *testing.T) { mockStore.EXPECT().Get(ctx, someHash).Return(nil, types.ErrKeyNotFound) mockIpniFinder.EXPECT().Find(ctx, someHash).Return(nil, errors.New("some error")) - _, err := providerIndex.getProviderResults(ctx, someHash) + _, err := providerIndex.getProviderResults(ctx, someHash, []multicodec.Code{0}) require.Error(t, err) }) @@ -163,7 +198,7 @@ func TestGetProviderResults(t *testing.T) { mockIpniFinder.EXPECT().Find(ctx, someHash).Return(&model.FindResponse{}, nil) mockLegacyClaims.EXPECT().Find(ctx, someHash).Return(nil, errors.New("some error")) - _, err := providerIndex.getProviderResults(ctx, someHash) + _, err := providerIndex.getProviderResults(ctx, someHash, []multicodec.Code{0}) require.Error(t, err) }) @@ -177,7 +212,7 @@ func TestGetProviderResults(t *testing.T) { providerIndex := New(mockStore, mockIpniFinder, mockIpniPublisher, mockLegacyClaims) someHash := testutil.RandomMultihash() - expectedResult := testutil.RandomProviderResult() + expectedResult := testutil.RandomLocationCommitmentProviderResult() ipniFinderResponse := &model.FindResponse{ MultihashResults: []model.MultihashResult{ { @@ -192,7 +227,7 @@ func TestGetProviderResults(t *testing.T) { mockIpniFinder.EXPECT().Find(ctx, someHash).Return(ipniFinderResponse, nil) mockStore.EXPECT().Set(ctx, someHash, []model.ProviderResult{expectedResult}, true).Return(errors.New("some error")) - _, err := providerIndex.getProviderResults(ctx, someHash) + _, err := providerIndex.getProviderResults(ctx, someHash, []multicodec.Code{metadata.LocationCommitmentID}) require.Error(t, err) })