analyze: Group CIDRs with --group

go.mod

@@ -10,18 +10,21 @@ require (
 	github.com/olekukonko/tablewriter v0.0.5
 	github.com/spf13/cobra v1.8.1
 	github.com/spf13/pflag v1.0.5
+	github.com/stretchr/testify v1.9.0
 	github.com/taoky/goaccessfmt v0.0.0-20240824074420-af31a41470aa
 	golang.org/x/sys v0.25.0
 )
 
 require (
+	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/itchyny/timefmt-go v0.1.6 // indirect
 	github.com/mattn/go-runewidth v0.0.16 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/rivo/uniseg v0.4.7 // indirect
-	github.com/stretchr/testify v1.9.0 // indirect
 	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -47,6 +47,7 @@ github.com/taoky/goaccessfmt v0.0.0-20240824074420-af31a41470aa/go.mod h1:TOGR4K
 golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34=
 golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=

pkg/analyze/analyze.go

@@ -31,7 +31,7 @@ type IPStats struct {
 	Requests uint64
 	LastURL  string
 
-	// Used at daemon mode only
+	// Used with daemon mode only
 	LastSize  uint64
 	FirstSeen time.Time
 
@@ -59,6 +59,24 @@ func (i IPStats) UpdateWith(item parser.LogItem) IPStats {
 	return i
 }
 
+func (i IPStats) MergeWith(other IPStats) IPStats {
+	i.Size += other.Size
+	i.Requests += other.Requests
+	if i.LastURL == other.LastURL {
+		if other.LastURLAccess.After(i.LastURLAccess) {
+			i.LastURLAccess = other.LastURLAccess
+		}
+		if other.LastURLUpdate.Before(i.LastURLUpdate) {
+			i.LastURLUpdate = other.LastURLUpdate
+		}
+	} else if other.LastURLUpdate.After(i.LastURLUpdate) {
+		i.LastURL = other.LastURL
+		i.LastURLUpdate = other.LastURLUpdate
+		i.LastURLAccess = other.LastURLAccess
+	}
+	return i
+}
+
 type StatKey struct {
 	Server string
 	Prefix netip.Prefix
@@ -77,6 +95,7 @@ type Analyzer struct {
 
 type AnalyzerConfig struct {
 	Absolute   bool
+	Group      bool
 	LogOutput  string
 	NoNetstat  bool
 	Parser     string
@@ -96,6 +115,7 @@ type AnalyzerConfig struct {
 
 func (c *AnalyzerConfig) InstallFlags(flags *pflag.FlagSet) {
 	flags.BoolVarP(&c.Absolute, "absolute", "a", c.Absolute, "Show absolute time for each item")
+	flags.BoolVarP(&c.Group, "group", "g", c.Group, "Try to group CIDRs")
 	flags.StringVarP(&c.LogOutput, "outlog", "o", c.LogOutput, "Change log output file")
 	flags.BoolVarP(&c.NoNetstat, "no-netstat", "", c.NoNetstat, "Do not detect active connections")
 	flags.StringVarP(&c.Parser, "parser", "p", c.Parser, "Log parser (see \"ayano list parsers\")")
@@ -144,6 +164,9 @@ func NewAnalyzer(c AnalyzerConfig) (*Analyzer, error) {
 		c.LogOutput = f.Name()
 		logger.SetOutput(f)
 	}
+	if c.Analyze {
+		logger.SetFlags(log.Flags() &^ (log.Ldate | log.Ltime))
+	}
 
 	return &Analyzer{
 		Config:    c,
@@ -286,18 +309,8 @@ func (a *Analyzer) GetActiveConns(activeConn map[netip.Prefix]int) {
 	}
 }
 
-func (a *Analyzer) PrintTopValues(displayRecord map[netip.Prefix]time.Time, sortBy SortByFlag, serverFilter string) {
-	activeConn := make(map[netip.Prefix]int)
-	if !a.Config.NoNetstat {
-		a.GetActiveConns(activeConn)
-	}
-
-	if a.Config.UseLock() {
-		a.mu.Lock()
-		defer a.mu.Unlock()
-	}
-
-	// sort stats key by value
+// SortedKeys returns stat keys sorted by value
+func (a *Analyzer) SortedKeys(sortBy SortByFlag, serverFilter string) []StatKey {
 	keys := make([]StatKey, 0)
 	for s := range a.stats {
 		if s.Server != serverFilter {
@@ -309,6 +322,21 @@ func (a *Analyzer) PrintTopValues(displayRecord map[netip.Prefix]time.Time, sort
 	if sortFunc != nil {
 		slices.SortFunc(keys, sortFunc)
 	}
+	return keys
+}
+
+func (a *Analyzer) PrintTopValues(displayRecord map[netip.Prefix]time.Time, sortBy SortByFlag, serverFilter string) {
+	activeConn := make(map[netip.Prefix]int)
+	if !a.Config.NoNetstat {
+		a.GetActiveConns(activeConn)
+	}
+
+	if a.Config.UseLock() {
+		a.mu.Lock()
+		defer a.mu.Unlock()
+	}
+
+	keys := a.SortedKeys(sortBy, serverFilter)
 
 	// print top N
 	top := a.Config.TopN
@@ -319,6 +347,48 @@ func (a *Analyzer) PrintTopValues(displayRecord map[netip.Prefix]time.Time, sort
 		top = len(keys)
 	}
 
+	if a.Config.Group {
+		groupedKeys := make(map[StatKey]struct{})
+		for _, key := range keys {
+			if key.Prefix.Bits() == 0 {
+				continue
+			}
+			adjacentKey := StatKey{key.Server, AdjacentPrefix(key.Prefix)}
+			_, ok := groupedKeys[adjacentKey]
+			if !ok {
+				// would insert into groupedKeys
+				if len(groupedKeys) >= top {
+					break
+				}
+				groupedKeys[key] = struct{}{}
+			}
+			for ok {
+				newStat := a.stats[key].MergeWith(a.stats[adjacentKey])
+				mergedPrefix := netip.PrefixFrom(key.Prefix.Addr(), key.Prefix.Bits()-1).Masked()
+				newKey := StatKey{key.Server, mergedPrefix}
+
+				a.stats[newKey] = newStat
+				delete(a.stats, key)
+				delete(a.stats, adjacentKey)
+
+				groupedKeys[newKey] = struct{}{}
+				delete(groupedKeys, key)
+				delete(groupedKeys, adjacentKey)
+
+				if newKey.Prefix.Bits() == 0 {
+					break
+				}
+				key = newKey
+				adjacentKey = StatKey{key.Server, AdjacentPrefix(key.Prefix)}
+				_, ok = groupedKeys[adjacentKey]
+			}
+		}
+		keys = a.SortedKeys(sortBy, serverFilter)
+		if len(keys) < top {
+			top = len(keys)
+		}
+	}
+
 	for i := range top {
 		key := keys[i]
 		ipStats := a.stats[key]

pkg/analyze/util.go

@@ -73,6 +73,23 @@ func (a *Analyzer) IPPrefix(ip netip.Addr) netip.Prefix {
 	return clientPrefix.Masked()
 }
 
+func AdjacentPrefix(p netip.Prefix) netip.Prefix {
+	bits := p.Bits()
+	if bits == 0 {
+		return p
+	}
+	a := p.Addr()
+	if a.Is4() {
+		addr := a.As4()
+		addr[(bits-1)/8] ^= uint8(1 << (7 - (bits-1)%8))
+		return netip.PrefixFrom(netip.AddrFrom4(addr), bits)
+	} else {
+		addr := a.As16()
+		addr[(bits-1)/8] ^= uint8(1 << (7 - (bits-1)%8))
+		return netip.PrefixFrom(netip.AddrFrom16(addr), bits)
+	}
+}
+
 func TruncateURLPath(input string) string {
 	count := strings.Count(input, "/")
 	if count <= 2 {

pkg/analyze/utils_test.go

@@ -0,0 +1,20 @@
+package analyze
+
+import (
+	"net/netip"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func assertAdjacentPrefix(t *testing.T, p1, p2 string) {
+	assert.Equal(t, netip.MustParsePrefix(p2), AdjacentPrefix(netip.MustParsePrefix(p1)))
+	assert.Equal(t, netip.MustParsePrefix(p1), AdjacentPrefix(netip.MustParsePrefix(p2)))
+}
+
+func TestAdjacentPrefix(t *testing.T) {
+	assertAdjacentPrefix(t, "1.2.3.0/24", "1.2.2.0/24")
+	assertAdjacentPrefix(t, "1.2.2.0/23", "1.2.0.0/23")
+	assertAdjacentPrefix(t, "2001:db8:114:514::/64", "2001:db8:114:515::/64")
+	assertAdjacentPrefix(t, "2001:db8:114:514::/63", "2001:db8:114:516::/63")
+}