Add network connectivity to health-checks

peers/app_request_network.go

@@ -2,13 +2,16 @@
 // See the file LICENSE for licensing terms.
 
 //go:generate mockgen -source=$GOFILE -destination=./mocks/mock_app_request_network.go -package=mocks
+//go:generate mockgen -destination=./avago_mocks/mock_network.go -package=avago_mocks github.com/ava-labs/avalanchego/network Network
 
 package peers
 
 import (
 	"context"
 	"encoding/hex"
+	"errors"
 	"fmt"
+	"math/big"
 	"sync"
 	"time"
 
@@ -28,6 +31,9 @@ import (
 	"github.com/ava-labs/avalanchego/vms/platformvm/warp"
 	"github.com/ava-labs/icm-services/peers/utils"
 	"github.com/ava-labs/icm-services/peers/validators"
+	subnetWarp "github.com/ava-labs/subnet-evm/precompile/contracts/warp"
+
+	sharedUtils "github.com/ava-labs/icm-services/utils"
 	"github.com/prometheus/client_golang/prometheus"
 	"go.uber.org/zap"
 )
@@ -39,8 +45,12 @@ const (
 	NumBootstrapNodes         = 5
 )
 
+var (
+	errNotEnoughConnectedStake = errors.New("failed to connect to a threshold of stake")
+)
+
 type AppRequestNetwork interface {
-	ConnectToCanonicalValidators(subnetID ids.ID) (
+	GetConnectedCanonicalValidators(subnetID ids.ID) (
 		*ConnectedCanonicalValidators,
 		error,
 	)
@@ -66,7 +76,7 @@ type appRequestNetwork struct {
 	infoAPI         *InfoAPI
 	logger          logging.Logger
 	lock            *sync.RWMutex
-	validatorClient *validators.CanonicalValidatorClient
+	validatorClient validators.CanonicalValidatorState
 	metrics         *AppRequestNetworkMetrics
 
 	trackedSubnets set.Set[ids.ID]
@@ -233,6 +243,8 @@ func (n *appRequestNetwork) containsSubnet(subnetID ids.ID) bool {
 	return n.trackedSubnets.Contains(subnetID)
 }
 
+// TrackSubnet adds the subnet to the list of tracked subnets
+// and initiates the connections to the subnet's validators asynchronously
 func (n *appRequestNetwork) TrackSubnet(subnetID ids.ID) {
 	if n.containsSubnet(subnetID) {
 		return
@@ -325,12 +337,9 @@ func (c *ConnectedCanonicalValidators) GetValidator(nodeID ids.NodeID) (*warp.Va
 	return c.ValidatorSet[c.NodeValidatorIndexMap[nodeID]], c.NodeValidatorIndexMap[nodeID]
 }
 
-// ConnectToCanonicalValidators connects to the canonical validators of the given subnet and returns the connected
-// validator information
-func (n *appRequestNetwork) ConnectToCanonicalValidators(subnetID ids.ID) (*ConnectedCanonicalValidators, error) {
-	// Track the subnet
-	n.TrackSubnet(subnetID)
-
+// GetConnectedCanonicalValidators returns the connected validator information for the given subnet
+// at the time of the call.
+func (n *appRequestNetwork) GetConnectedCanonicalValidators(subnetID ids.ID) (*ConnectedCanonicalValidators, error) {
 	// Get the subnet's current canonical validator set
 	startPChainAPICall := time.Now()
 	validatorSet, totalValidatorWeight, err := n.validatorClient.GetCurrentCanonicalValidatorSet(subnetID)
@@ -351,8 +360,16 @@ func (n *appRequestNetwork) ConnectToCanonicalValidators(subnetID ids.ID) (*Conn
 		}
 	}
 
+	peerInfo := n.network.PeerInfo(nodeIDs.List())
+	connectedPeers := set.NewSet[ids.NodeID](len(nodeIDs))
+	for _, peer := range peerInfo {
+		if nodeIDs.Contains(peer.ID) {
+			connectedPeers.Add(peer.ID)
+		}
+	}
+
 	// Calculate the total weight of connected validators.
-	connectedWeight := calculateConnectedWeight(validatorSet, nodeValidatorIndexMap, nodeIDs)
+	connectedWeight := calculateConnectedWeight(validatorSet, nodeValidatorIndexMap, connectedPeers)
 
 	return &ConnectedCanonicalValidators{
 		ConnectedWeight:       connectedWeight,
@@ -391,6 +408,26 @@ func (n *appRequestNetwork) setPChainAPICallLatencyMS(latency float64) {
 
 // Non-receiver util functions
 
+func GetNetworkHealthFunc(network AppRequestNetwork, subnetIDs []ids.ID) func(context.Context) error {
+	return func(context.Context) error {
+		for _, subnetID := range subnetIDs {
+			connectedValidators, err := network.GetConnectedCanonicalValidators(subnetID)
+			if err != nil {
+				return fmt.Errorf(
+					"failed to get connected validators: %s, %w", subnetID, err)
+			}
+			if !sharedUtils.CheckStakeWeightExceedsThreshold(
+				big.NewInt(0).SetUint64(connectedValidators.ConnectedWeight),
+				connectedValidators.TotalValidatorWeight,
+				subnetWarp.WarpDefaultQuorumNumerator,
+			) {
+				return errNotEnoughConnectedStake
+			}
+		}
+		return nil
+	}
+}
+
 func calculateConnectedWeight(
 	validatorSet []*warp.Validator,
 	nodeValidatorIndexMap map[ids.NodeID]int,

peers/app_request_network_test.go

@@ -7,10 +7,15 @@ import (
 	"testing"
 
 	"github.com/ava-labs/avalanchego/ids"
+	"github.com/ava-labs/avalanchego/network/peer"
 	"github.com/ava-labs/avalanchego/utils/crypto/bls"
 	"github.com/ava-labs/avalanchego/utils/set"
 	"github.com/ava-labs/avalanchego/vms/platformvm/warp"
+	"github.com/ava-labs/icm-services/peers/avago_mocks"
+	validator_mocks "github.com/ava-labs/icm-services/peers/validators/mocks"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/stretchr/testify/require"
+	"go.uber.org/mock/gomock"
 )
 
 func TestCalculateConnectedWeight(t *testing.T) {
@@ -48,6 +53,110 @@ func TestCalculateConnectedWeight(t *testing.T) {
 	require.Equal(t, uint64(60), connectedWeight3)
 }
 
+func TestConnectToCanonicalValidators(t *testing.T) {
+	ctrl := gomock.NewController(t)
+
+	subnetID := ids.GenerateTestID()
+	validator1_1 := makeValidator(t, 1, 1)
+	validator2_1 := makeValidator(t, 2, 1)
+	validator3_2 := makeValidator(t, 3, 2)
+	metrics, _ := newAppRequestNetworkMetrics(prometheus.DefaultRegisterer)
+
+	testCases := []struct {
+		name                    string
+		validators              []*warp.Validator
+		connectedNodes          []ids.NodeID
+		expectedConnectedWeight uint64
+		expectedTotalWeight     uint64
+	}{
+		{
+			name:                    "no connected nodes, one validator",
+			validators:              []*warp.Validator{&validator1_1},
+			connectedNodes:          []ids.NodeID{},
+			expectedConnectedWeight: 0,
+			expectedTotalWeight:     1,
+		},
+		{
+			name:       "all validators, missing one nodeID",
+			validators: []*warp.Validator{&validator1_1, &validator2_1, &validator3_2},
+			connectedNodes: []ids.NodeID{
+				validator1_1.NodeIDs[0],
+				validator2_1.NodeIDs[0],
+				validator3_2.NodeIDs[0],
+				validator3_2.NodeIDs[1],
+			},
+			expectedConnectedWeight: 6,
+			expectedTotalWeight:     6,
+		},
+		{
+			name:       "fully connected",
+			validators: []*warp.Validator{&validator1_1, &validator2_1, &validator3_2},
+			connectedNodes: []ids.NodeID{
+				validator1_1.NodeIDs[0],
+				validator2_1.NodeIDs[0],
+				validator3_2.NodeIDs[0],
+				validator3_2.NodeIDs[1],
+			},
+			expectedConnectedWeight: 6,
+			expectedTotalWeight:     6,
+		},
+		{
+			name:       "missing conn to double node validator",
+			validators: []*warp.Validator{&validator1_1, &validator2_1, &validator3_2},
+			connectedNodes: []ids.NodeID{
+				validator1_1.NodeIDs[0],
+				validator2_1.NodeIDs[0],
+			},
+			expectedConnectedWeight: 3,
+			expectedTotalWeight:     6,
+		},
+		{
+			name:       "irrelevant nodes",
+			validators: []*warp.Validator{&validator1_1, &validator2_1},
+			connectedNodes: []ids.NodeID{
+				validator1_1.NodeIDs[0],
+				validator2_1.NodeIDs[0],
+				validator3_2.NodeIDs[0], // this nodeID does not map to the validator
+			},
+			expectedConnectedWeight: 3,
+			expectedTotalWeight:     3,
+		},
+	}
+	for _, testCase := range testCases {
+		t.Run(testCase.name, func(t *testing.T) {
+			mockNetwork := avago_mocks.NewMockNetwork(ctrl)
+			mockValidatorClient := validator_mocks.NewMockCanonicalValidatorState(ctrl)
+			arNetwork := appRequestNetwork{
+				network:         mockNetwork,
+				validatorClient: mockValidatorClient,
+				metrics:         metrics,
+			}
+			var totalWeight uint64
+			for _, vdr := range testCase.validators {
+				totalWeight += vdr.Weight
+			}
+			mockValidatorClient.EXPECT().GetCurrentCanonicalValidatorSet(subnetID).Return(
+				testCase.validators,
+				totalWeight,
+				nil,
+			).Times(1)
+
+			peerInfo := make([]peer.Info, len(testCase.validators))
+			for _, node := range testCase.connectedNodes {
+				peerInfo = append(peerInfo, peer.Info{
+					ID: node,
+				})
+			}
+			mockNetwork.EXPECT().PeerInfo(gomock.Any()).Return(peerInfo).Times(1)
+
+			ret, err := arNetwork.GetConnectedCanonicalValidators(subnetID)
+			require.Equal(t, testCase.expectedConnectedWeight, ret.ConnectedWeight)
+			require.Equal(t, testCase.expectedTotalWeight, ret.TotalValidatorWeight)
+			require.NoError(t, err)
+		})
+	}
+}
+
 func makeValidator(t *testing.T, weight uint64, numNodeIDs int) warp.Validator {
 	sk, err := bls.NewSecretKey()
 	require.NoError(t, err)