Skip to content

Commit

Permalink
fix: fetch from legacy if there are no interesting claims in IPNI (#61)
Browse files Browse the repository at this point in the history 
With the current implementation, we only look in legacy claims storage
if IPNI returns no results.

This PR changes this so that we also search in legacy claims when IPNI
return results that are not interesting for the current query.
  • Loading branch information
@volmedo
volmedo authored Dec 16, 2024
1 parent 05441c2 commit 1dd0663
Show file tree
Hide file tree
Showing 3 changed files with 113 additions and 30 deletions.
24 changes: 24 additions & 0 deletions pkg/internal/testutil/gen.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,14 @@ import (
"github.com/ipld/go-ipld-prime/datamodel"
cidlink "github.com/ipld/go-ipld-prime/linking/cid"
"github.com/ipni/go-libipni/find/model"
ipnimeta "github.com/ipni/go-libipni/metadata"
crypto "github.com/libp2p/go-libp2p/core/crypto"
peer "github.com/libp2p/go-libp2p/core/peer"
"github.com/multiformats/go-multiaddr"
manet "github.com/multiformats/go-multiaddr/net"
mh "github.com/multiformats/go-multihash"
"github.com/storacha/go-capabilities/pkg/assert"
"github.com/storacha/go-metadata"
"github.com/storacha/go-ucanto/core/car"
"github.com/storacha/go-ucanto/core/delegation"
"github.com/storacha/go-ucanto/core/ipld/block"
Expand Down Expand Up @@ -172,6 +174,28 @@ func RandomProviderResult() model.ProviderResult {
}
}

func RandomBitswapProviderResult() model.ProviderResult {
pr := RandomProviderResult()
bitswapMeta, _ := ipnimeta.Bitswap{}.MarshalBinary()
pr.Metadata = bitswapMeta
return pr
}

func RandomLocationCommitmentProviderResult() model.ProviderResult {
shard := RandomCID().(cidlink.Link).Cid
locationMeta := metadata.LocationCommitmentMetadata{
Shard: &shard,
Range: &metadata.Range{Offset: 128},
Expiration: 0,
Claim: RandomCID().(cidlink.Link).Cid,
}
metaBytes, _ := locationMeta.MarshalBinary()

pr := RandomProviderResult()
pr.Metadata = metaBytes
return pr
}

func RandomShardedDagIndexView(size int) (mh.Multihash, blobindex.ShardedDagIndexView) {
root, digest, bytes := RandomCAR(size)
shard, err := blobindex.FromShardArchives(root, [][]byte{bytes})
Expand Down
60 changes: 42 additions & 18 deletions pkg/service/providerindex/providerindex.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,18 +58,15 @@ func New(providerStore types.ProviderStore, findClient ipnifind.Finder, publishe
// encodedcontextid's by hashing space DID and Hash, and filter for a matching context id
// Future TODO: kick off a conversion task to update the records
func (pi *ProviderIndexService) Find(ctx context.Context, qk QueryKey) ([]model.ProviderResult, error) {
results, err := pi.getProviderResults(ctx, qk.Hash)
results, err := pi.getProviderResults(ctx, qk.Hash, qk.TargetClaims)
if err != nil {
return nil, err
}
results, err = pi.filteredCodecs(results, qk.TargetClaims)
if err != nil {
return nil, err
}
return pi.filterBySpace(results, qk.Hash, qk.Spaces)

return filterBySpace(results, qk.Hash, qk.Spaces)
}

func (pi *ProviderIndexService) getProviderResults(ctx context.Context, mh mh.Multihash) ([]model.ProviderResult, error) {
func (pi *ProviderIndexService) getProviderResults(ctx context.Context, mh mh.Multihash, targetClaims []multicodec.Code) ([]model.ProviderResult, error) {
res, err := pi.providerStore.Get(ctx, mh)
if err == nil {
return res, nil
Expand All @@ -78,20 +75,14 @@ func (pi *ProviderIndexService) getProviderResults(ctx context.Context, mh mh.Mu
return nil, err
}

var results []model.ProviderResult

findRes, err := pi.findClient.Find(ctx, mh)
results, err := pi.fetchFromIPNI(ctx, mh, targetClaims)
if err != nil {
return nil, err
}

for _, mhres := range findRes.MultihashResults {
results = append(results, mhres.ProviderResults...)
}

// try legacy claims storage if nothing was found on IPNI
// if nothing was found on IPNI, try legacy claims storage
if len(results) == 0 {
legacyResults, err := pi.legacyClaims.Find(ctx, mh)
legacyResults, err := pi.fetchFromLegacy(ctx, mh, targetClaims)
if err != nil {
return nil, err
}
Expand All @@ -109,7 +100,40 @@ func (pi *ProviderIndexService) getProviderResults(ctx context.Context, mh mh.Mu
return results, nil
}

func (pi *ProviderIndexService) filteredCodecs(results []model.ProviderResult, codecs []multicodec.Code) ([]model.ProviderResult, error) {
func (pi *ProviderIndexService) fetchFromIPNI(ctx context.Context, mh mh.Multihash, targetClaims []multicodec.Code) ([]model.ProviderResult, error) {
var results []model.ProviderResult
findRes, err := pi.findClient.Find(ctx, mh)
if err != nil {
return nil, err
}

for _, mhres := range findRes.MultihashResults {
results = append(results, mhres.ProviderResults...)
}

results, err = filterCodecs(results, targetClaims)
if err != nil {
return nil, err
}

return results, nil
}

func (pi *ProviderIndexService) fetchFromLegacy(ctx context.Context, mh mh.Multihash, targetClaims []multicodec.Code) ([]model.ProviderResult, error) {
results, err := pi.legacyClaims.Find(ctx, mh)
if err != nil {
return nil, err
}

results, err = filterCodecs(results, targetClaims)
if err != nil {
return nil, err
}

return results, nil
}

func filterCodecs(results []model.ProviderResult, codecs []multicodec.Code) ([]model.ProviderResult, error) {
if len(codecs) == 0 {
return results, nil
}
Expand All @@ -127,7 +151,7 @@ func (pi *ProviderIndexService) filteredCodecs(results []model.ProviderResult, c
})
}

func (pi *ProviderIndexService) filterBySpace(results []model.ProviderResult, mh mh.Multihash, spaces []did.DID) ([]model.ProviderResult, error) {
func filterBySpace(results []model.ProviderResult, mh mh.Multihash, spaces []did.DID) ([]model.ProviderResult, error) {
if len(spaces) == 0 {
return results, nil
}
Expand Down
59 changes: 47 additions & 12 deletions pkg/service/providerindex/providerindex_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ import (
"testing"

"github.com/ipni/go-libipni/find/model"
"github.com/multiformats/go-multicodec"
"github.com/storacha/go-metadata"
"github.com/storacha/indexing-service/pkg/internal/testutil"
"github.com/storacha/indexing-service/pkg/internal/testutil/mocks"
"github.com/storacha/indexing-service/pkg/types"
Expand All @@ -28,7 +30,7 @@ func TestGetProviderResults(t *testing.T) {

mockStore.EXPECT().Get(ctx, someHash).Return([]model.ProviderResult{expectedResult}, nil)

results, err := providerIndex.getProviderResults(ctx, someHash)
results, err := providerIndex.getProviderResults(ctx, someHash, []multicodec.Code{0})

require.NoError(t, err)
require.Equal(t, []model.ProviderResult{expectedResult}, results)
Expand All @@ -43,7 +45,7 @@ func TestGetProviderResults(t *testing.T) {
providerIndex := New(mockStore, mockIpniFinder, mockIpniPublisher, mockLegacyClaims)

someHash := testutil.RandomMultihash()
expectedResult := testutil.RandomProviderResult()
expectedResult := testutil.RandomLocationCommitmentProviderResult()
ipniFinderResponse := &model.FindResponse{
MultihashResults: []model.MultihashResult{
{
Expand All @@ -59,13 +61,13 @@ func TestGetProviderResults(t *testing.T) {
mockIpniFinder.EXPECT().Find(ctx, someHash).Return(ipniFinderResponse, nil)
mockStore.EXPECT().Set(ctx, someHash, []model.ProviderResult{expectedResult}, true).Return(nil)

results, err := providerIndex.getProviderResults(ctx, someHash)
results, err := providerIndex.getProviderResults(ctx, someHash, []multicodec.Code{metadata.LocationCommitmentID})

require.NoError(t, err)
require.Equal(t, []model.ProviderResult{expectedResult}, results)
})

t.Run("results not cached, not found in IPNI, found in legacy claims service, results cached afterwards", func(t *testing.T) {
t.Run("results not cached, no results from IPNI, found in legacy claims service, results cached afterwards", func(t *testing.T) {
mockStore := mocks.NewMockProviderStore(t)
mockIpniFinder := mocks.NewMockFinder(t)
mockIpniPublisher := mocks.NewMockPublisher(t)
Expand All @@ -74,7 +76,7 @@ func TestGetProviderResults(t *testing.T) {
providerIndex := New(mockStore, mockIpniFinder, mockIpniPublisher, mockLegacyClaims)

someHash := testutil.RandomMultihash()
expectedResult := testutil.RandomProviderResult()
expectedResult := testutil.RandomLocationCommitmentProviderResult()

ctx := context.Background()

Expand All @@ -83,7 +85,40 @@ func TestGetProviderResults(t *testing.T) {
mockLegacyClaims.EXPECT().Find(ctx, someHash).Return([]model.ProviderResult{expectedResult}, nil)
mockStore.EXPECT().Set(ctx, someHash, []model.ProviderResult{expectedResult}, true).Return(nil)

results, err := providerIndex.getProviderResults(ctx, someHash)
results, err := providerIndex.getProviderResults(ctx, someHash, []multicodec.Code{metadata.LocationCommitmentID})

require.NoError(t, err)
require.Equal(t, []model.ProviderResult{expectedResult}, results)
})

t.Run("results not cached, IPNI returns uninteresting results, search in legacy claims", func(t *testing.T) {
mockStore := mocks.NewMockProviderStore(t)
mockIpniFinder := mocks.NewMockFinder(t)
mockIpniPublisher := mocks.NewMockPublisher(t)
mockLegacyClaims := mocks.NewMockLegacyClaimsFinder(t)

providerIndex := New(mockStore, mockIpniFinder, mockIpniPublisher, mockLegacyClaims)

someHash := testutil.RandomMultihash()
bitswapResult := testutil.RandomBitswapProviderResult()
ipniFinderResponse := &model.FindResponse{
MultihashResults: []model.MultihashResult{
{
Multihash: someHash,
ProviderResults: []model.ProviderResult{bitswapResult},
},
},
}
expectedResult := testutil.RandomLocationCommitmentProviderResult()

ctx := context.Background()

mockStore.EXPECT().Get(ctx, someHash).Return(nil, types.ErrKeyNotFound)
mockIpniFinder.EXPECT().Find(ctx, someHash).Return(ipniFinderResponse, nil)
mockLegacyClaims.EXPECT().Find(ctx, someHash).Return([]model.ProviderResult{expectedResult}, nil)
mockStore.EXPECT().Set(ctx, someHash, []model.ProviderResult{expectedResult}, true).Return(nil)

results, err := providerIndex.getProviderResults(ctx, someHash, []multicodec.Code{metadata.LocationCommitmentID})

require.NoError(t, err)
require.Equal(t, []model.ProviderResult{expectedResult}, results)
Expand All @@ -105,7 +140,7 @@ func TestGetProviderResults(t *testing.T) {
mockIpniFinder.EXPECT().Find(ctx, someHash).Return(&model.FindResponse{}, nil)
mockLegacyClaims.EXPECT().Find(ctx, someHash).Return([]model.ProviderResult{}, nil)

results, err := providerIndex.getProviderResults(ctx, someHash)
results, err := providerIndex.getProviderResults(ctx, someHash, []multicodec.Code{0})

require.NoError(t, err)
require.Empty(t, results)
Expand All @@ -124,7 +159,7 @@ func TestGetProviderResults(t *testing.T) {
ctx := context.Background()
mockStore.EXPECT().Get(ctx, someHash).Return(nil, errors.New("some error"))

_, err := providerIndex.getProviderResults(ctx, someHash)
_, err := providerIndex.getProviderResults(ctx, someHash, []multicodec.Code{0})

require.Error(t, err)
})
Expand All @@ -143,7 +178,7 @@ func TestGetProviderResults(t *testing.T) {
mockStore.EXPECT().Get(ctx, someHash).Return(nil, types.ErrKeyNotFound)
mockIpniFinder.EXPECT().Find(ctx, someHash).Return(nil, errors.New("some error"))

_, err := providerIndex.getProviderResults(ctx, someHash)
_, err := providerIndex.getProviderResults(ctx, someHash, []multicodec.Code{0})

require.Error(t, err)
})
Expand All @@ -163,7 +198,7 @@ func TestGetProviderResults(t *testing.T) {
mockIpniFinder.EXPECT().Find(ctx, someHash).Return(&model.FindResponse{}, nil)
mockLegacyClaims.EXPECT().Find(ctx, someHash).Return(nil, errors.New("some error"))

_, err := providerIndex.getProviderResults(ctx, someHash)
_, err := providerIndex.getProviderResults(ctx, someHash, []multicodec.Code{0})

require.Error(t, err)
})
Expand All @@ -177,7 +212,7 @@ func TestGetProviderResults(t *testing.T) {
providerIndex := New(mockStore, mockIpniFinder, mockIpniPublisher, mockLegacyClaims)

someHash := testutil.RandomMultihash()
expectedResult := testutil.RandomProviderResult()
expectedResult := testutil.RandomLocationCommitmentProviderResult()
ipniFinderResponse := &model.FindResponse{
MultihashResults: []model.MultihashResult{
{
Expand All @@ -192,7 +227,7 @@ func TestGetProviderResults(t *testing.T) {
mockIpniFinder.EXPECT().Find(ctx, someHash).Return(ipniFinderResponse, nil)
mockStore.EXPECT().Set(ctx, someHash, []model.ProviderResult{expectedResult}, true).Return(errors.New("some error"))

_, err := providerIndex.getProviderResults(ctx, someHash)
_, err := providerIndex.getProviderResults(ctx, someHash, []multicodec.Code{metadata.LocationCommitmentID})

require.Error(t, err)
})
Expand Down

0 comments on commit 1dd0663

Please # to comment.