telemetry: add TCP RTT info collection

go.mod

@@ -34,7 +34,7 @@ require (
 	github.com/spf13/cobra v0.0.3
 	github.com/stretchr/testify v1.7.1
 	golang.org/x/crypto v0.0.0-20220321153916-2c7772ba3064
-	golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8
+	golang.org/x/sys v0.1.0
 	golang.org/x/text v0.3.7
 	gopkg.in/sohlich/elogrus.v3 v3.0.0-20180410122755-1fa29e2f2009
 )

go.sum

@@ -188,8 +188,8 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8 h1:OH54vjqzRWmbJ62fjuhxy7AxFFgoHN0/DPc/UrL8cAs=
-golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U=
+golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

logging/telemetryspec/event.go

@@ -302,6 +302,9 @@ type PeerConnectionDetails struct {
 	MessageDelay int64 `json:",omitempty"`
 	// DuplicateFilterCount is the number of times this peer has sent us a message hash to filter that it had already sent before.
 	DuplicateFilterCount uint64
+	// RTT and RTTVar are latency observations from TCP.
+	RTT    int64 `json:",omitempty"`
+	RTTVar int64 `json:",omitempty"`
 }
 
 // CatchpointGenerationEvent event

network/wsNetwork.go

@@ -1830,6 +1830,11 @@ func (wn *WebsocketNetwork) sendPeerConnectionsTelemetryStatus() {
 			connDetail.Address = peer.OriginAddress()
 			connectionDetails.IncomingPeers = append(connectionDetails.IncomingPeers, connDetail)
 		}
+		rttInfo, err := util.GetConnRTT(peer.conn.UnderlyingConn())
+		if err == nil {
+			connDetail.RTT = int64(rttInfo.RTT)
+			connDetail.RTTVar = int64(rttInfo.RTTVar)
+		}
 	}
 
 	wn.log.EventWithDetails(telemetryspec.Network, telemetryspec.PeerConnectionsEvent, connectionDetails)

network/wsNetwork_test.go

@@ -721,29 +721,15 @@ func TestAddrToGossipAddr(t *testing.T) {
 
 type nopConn struct{}
 
-func (nc *nopConn) RemoteAddr() net.Addr {
-	return nil
-}
-func (nc *nopConn) NextReader() (int, io.Reader, error) {
-	return 0, nil, nil
-}
-func (nc *nopConn) WriteMessage(int, []byte) error {
-	return nil
-}
-func (nc *nopConn) WriteControl(int, []byte, time.Time) error {
-	return nil
-}
-func (nc *nopConn) SetReadLimit(limit int64) {
-}
-func (nc *nopConn) CloseWithoutFlush() error {
-	return nil
-}
-func (nc *nopConn) SetPingHandler(h func(appData string) error) {
-
-}
-func (nc *nopConn) SetPongHandler(h func(appData string) error) {
-
-}
+func (nc *nopConn) RemoteAddr() net.Addr                        { return nil }
+func (nc *nopConn) NextReader() (int, io.Reader, error)         { return 0, nil, nil }
+func (nc *nopConn) WriteMessage(int, []byte) error              { return nil }
+func (nc *nopConn) WriteControl(int, []byte, time.Time) error   { return nil }
+func (nc *nopConn) SetReadLimit(limit int64)                    {}
+func (nc *nopConn) CloseWithoutFlush() error                    { return nil }
+func (nc *nopConn) SetPingHandler(h func(appData string) error) {}
+func (nc *nopConn) SetPongHandler(h func(appData string) error) {}
+func (nc *nopConn) UnderlyingConn() net.Conn                    { return nil }
 
 var nopConnSingleton = nopConn{}
 

network/wsPeer.go

@@ -124,6 +124,7 @@ type wsPeerWebsocketConn interface {
 	CloseWithoutFlush() error
 	SetPingHandler(h func(appData string) error)
 	SetPongHandler(h func(appData string) error)
+	UnderlyingConn() net.Conn
 }
 
 type sendMessage struct {

util/rtt.go

@@ -0,0 +1,42 @@
+// Copyright (C) 2019-2022 Algorand, Inc.
+// This file is part of go-algorand
+//
+// go-algorand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// go-algorand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with go-algorand.  If not, see <https://www.gnu.org/licenses/>.
+
+package util
+
+import (
+	"errors"
+	"net"
+)
+
+// RTTInfo provides smoothed RTT and RTT variance from socket-level TCP information.
+type RTTInfo struct {
+	RTT    uint32
+	RTTVar uint32
+}
+
+var (
+	// ErrNotSyscallConn is reported when GetConnRTT is passed a connection that doesn't satisfy the syscall.Conn interface.
+	ErrNotSyscallConn = errors.New("conn doesn't satisfy syscall.Conn")
+	// ErrRTTUnsupported is reported if TCP information is not available for this platform.
+	ErrRTTUnsupported = errors.New("GetConnRTT not supported on this platform")
+)
+
+// GetConnRTT returns RTT statistics for a TCP connection collected by the
+// underlying network implementation, using a system call on Linux and Mac
+// and returning an error for unsupported platforms.
+func GetConnRTT(conn net.Conn) (*RTTInfo, error) {
+	return getConnRTT(conn)
+}

util/rtt_darwin.go

@@ -0,0 +1,53 @@
+// Copyright (C) 2019-2022 Algorand, Inc.
+// This file is part of go-algorand
+//
+// go-algorand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// go-algorand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with go-algorand.  If not, see <https://www.gnu.org/licenses/>.
+
+package util
+
+import (
+	"net"
+	"syscall"
+
+	"golang.org/x/sys/unix"
+)
+
+func getConnRTT(conn net.Conn) (*RTTInfo, error) {
+	sysconn, ok := conn.(syscall.Conn)
+	if !ok {
+		return nil, ErrNotSyscallConn
+	}
+	raw, err := sysconn.SyscallConn()
+	if err != nil {
+		return nil, err
+	}
+
+	var info *unix.TCPConnectionInfo
+	var getSockoptErr error
+	err = raw.Control(func(fd uintptr) {
+		info, getSockoptErr = unix.GetsockoptTCPConnectionInfo(int(fd), unix.IPPROTO_TCP, unix.TCP_CONNECTION_INFO)
+	})
+	if err != nil {
+		return nil, err
+	}
+	if getSockoptErr != nil {
+		return nil, getSockoptErr
+	}
+	var ret RTTInfo
+	if info != nil {
+		ret.RTT = info.Srtt
+		ret.RTTVar = info.Rttvar
+	}
+	return &ret, nil
+}

util/rtt_linux.go

@@ -0,0 +1,53 @@
+// Copyright (C) 2019-2022 Algorand, Inc.
+// This file is part of go-algorand
+//
+// go-algorand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// go-algorand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with go-algorand.  If not, see <https://www.gnu.org/licenses/>.
+
+package util
+
+import (
+	"net"
+	"syscall"
+
+	"golang.org/x/sys/unix"
+)
+
+func getConnRTT(conn net.Conn) (*RTTInfo, error) {
+	sysconn, ok := conn.(syscall.Conn)
+	if !ok {
+		return nil, ErrNotSyscallConn
+	}
+	raw, err := sysconn.SyscallConn()
+	if err != nil {
+		return nil, err
+	}
+
+	var info *unix.TCPInfo
+	var getSockoptErr error
+	err = raw.Control(func(fd uintptr) {
+		info, getSockoptErr = unix.GetsockoptTCPInfo(int(fd), unix.IPPROTO_TCP, unix.TCP_INFO)
+	})
+	if err != nil {
+		return nil, err
+	}
+	if getSockoptErr != nil {
+		return nil, getSockoptErr
+	}
+	var ret RTTInfo
+	if info != nil {
+		ret.RTT = info.Rtt
+		ret.RTTVar = info.Rttvar
+	}
+	return &ret, nil
+}

util/rtt_noop.go

@@ -0,0 +1,26 @@
+// Copyright (C) 2019-2022 Algorand, Inc.
+// This file is part of go-algorand
+//
+// go-algorand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// go-algorand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with go-algorand.  If not, see <https://www.gnu.org/licenses/>.
+
+//go:build !linux && !darwin
+// +build !linux,!darwin
+
+package util
+
+import "net"
+
+func getConnRTT(conn net.Conn) (*RTTInfo, error) {
+	return nil, ErrRTTUnsupported
+}