Skip to content

Commit

Permalink
Merge pull request #36 from satta/rdns
Browse files Browse the repository at this point in the history
implement active reverse DNS enrichment
  • Loading branch information
satta authored Mar 1, 2019
2 parents 1a3de5b + 210464c commit f82a9c5
Show file tree
Hide file tree
Showing 10 changed files with 287 additions and 9 deletions.
2 changes: 1 addition & 1 deletion LICENSE
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
Copyright (c) 2017, 2018, DCSO Deutsche Cyber-Sicherheitsorganisation GmbH
Copyright (c) 2017, 2018, 2019, DCSO Deutsche Cyber-Sicherheitsorganisation GmbH
All rights reserved.

Redistribution and use in source and binary forms, with or without
Expand Down
17 changes: 12 additions & 5 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,10 @@ Usage:
fever run [flags]
Flags:
--active-rdns enable active rDNS enrichment for src/dst IPs
--active-rdns-cache-expiry duration cache expiry interval for rDNS lookups (default 2m0s)
--active-rdns-private-only only do active rDNS enrichment for RFC1918 IPs
--bloom-alert-prefix string String prefix for Bloom filter alerts (default "BLF")
-b, --bloom-file string Bloom filter for external indicator screening
-z, --bloom-zipped use gzipped Bloom filter file
-c, --chunksize uint chunk size for batched event handling (e.g. inserts) (default 50000)
Expand All @@ -42,11 +46,11 @@ Flags:
--flowextract-bloom-selector string IP address Bloom filter to select flows to extract
--flowextract-enable extract and forward flow metadata
--flowextract-submission-exchange string Exchange to which raw flow events will be submitted (default "flows")
--flowextract-submission-url string URL to which raw flow events will be submitted
--flowextract-submission-url string URL to which raw flow events will be submitted (default "amqp://guest:guest@localhost:5672/")
-n, --flowreport-interval duration time interval for report submissions
--flowreport-nocompress send uncompressed flow reports (default is gzip)
--flowreport-submission-exchange string Exchange to which flow reports will be submitted (default "aggregations")
--flowreport-submission-url string URL to which flow reports will be submitted
--flowreport-submission-url string URL to which flow reports will be submitted (default "amqp://guest:guest@localhost:5672/")
--flushcount uint maximum number of events in one batch (e.g. for flow extraction) (default 100000)
-f, --flushtime duration time interval for event aggregation (default 1m0s)
-T, --fwd-all-types forward all event types
Expand All @@ -55,17 +59,20 @@ Flags:
-r, --in-redis string Redis input server (assumes "suricata" list key, no pwd)
--in-redis-nopipe do not use Redis pipelining
-i, --in-socket string filename of input socket (accepts EVE JSON) (default "/tmp/suri.sock")
--ip-alert-prefix string String prefix for IP blacklist alerts (default "IP-BLACKLIST")
--ip-blacklist string List with IP ranges to alert on
--logfile string Path to log file
--logjson Output logs in JSON format
--metrics-enable submit performance metrics to central sink
--metrics-submission-exchange string Exchange to which metrics will be submitted (default "metrics")
--metrics-submission-url string URL to which metrics will be submitted
-o, --out-socket string path to output socket (to forwarder) (default "/tmp/suri-forward.sock")
--metrics-submission-url string URL to which metrics will be submitted (default "amqp://guest:guest@localhost:5672/")
-o, --out-socket string path to output socket (to forwarder), empty string disables forwarding (default "/tmp/suri-forward.sock")
--pdns-enable collect and forward aggregated passive DNS data
--pdns-submission-exchange string Exchange to which passive DNS events will be submitted (default "pdns")
--pdns-submission-url string URL to which passive DNS events will be submitted
--pdns-submission-url string URL to which passive DNS events will be submitted (default "amqp://guest:guest@localhost:5672/")
--profile string enable runtime profiling to given file
--reconnect-retries uint number of retries connecting to socket or sink, 0 = no retry limit
--toolname string set toolname (default "fever")
-v, --verbose enable verbose logging (debug log level)
Global Flags:
Expand Down
19 changes: 18 additions & 1 deletion cmd/fever/cmds/run.go
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
package cmd

// DCSO FEVER
// Copyright (c) 2017, 2018, DCSO GmbH
// Copyright (c) 2017, 2018, 2019, DCSO GmbH

import (
"io"
Expand Down Expand Up @@ -179,6 +179,15 @@ func mainfunc(cmd *cobra.Command, args []string) {
if pse != nil {
forwardHandler.(*processing.ForwardHandler).SubmitStats(pse)
}
rdns := viper.GetBool("active.rdns")
if rdns {
expiryPeriod := viper.GetDuration("active.rdns-cache-expiry")
forwardHandler.(*processing.ForwardHandler).EnableRDNS(expiryPeriod)
privateOnly := viper.GetBool("active.rdns-private-only")
if privateOnly {
forwardHandler.(*processing.ForwardHandler).RDNSHandler.EnableOnlyPrivateIPRanges()
}
}
forwardHandler.(*processing.ForwardHandler).Run()
defer func() {
c := make(chan bool)
Expand Down Expand Up @@ -564,6 +573,14 @@ func init() {
runCmd.PersistentFlags().StringP("flowextract-submission-exchange", "", "flows", "Exchange to which raw flow events will be submitted")
viper.BindPFlag("flowextract.submission-exchange", runCmd.PersistentFlags().Lookup("flowextract-submission-exchange"))

// Active enrichment options
runCmd.PersistentFlags().BoolP("active-rdns", "", false, "enable active rDNS enrichment for src/dst IPs")
viper.BindPFlag("active.rdns", runCmd.PersistentFlags().Lookup("active-rdns"))
runCmd.PersistentFlags().DurationP("active-rdns-cache-expiry", "", 2*time.Minute, "cache expiry interval for rDNS lookups")
viper.BindPFlag("active.rdns-cache-expiry", runCmd.PersistentFlags().Lookup("active-rdns-cache-expiry"))
runCmd.PersistentFlags().BoolP("active-rdns-private-only", "", false, "only do active rDNS enrichment for RFC1918 IPs")
viper.BindPFlag("active.rdns-private-only", runCmd.PersistentFlags().Lookup("active-rdns-private-only"))

// Logging options
runCmd.PersistentFlags().StringP("logfile", "", "", "Path to log file")
viper.BindPFlag("logging.file", runCmd.PersistentFlags().Lookup("logfile"))
Expand Down
7 changes: 7 additions & 0 deletions fever.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,13 @@ flowextract:
# zipped: true
# alert-prefix: BLF

# Configuration for active information gathering.
active:
# Enable reverse DNS lookups for src/dst IPs.
rdns: false
rdns-private-only: true
rdns-cache-expiry: 120s

logging:
# Insert file name here to redirect logs to separate file.
file:
Expand Down
30 changes: 28 additions & 2 deletions processing/forward_handler.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ package processing
// Copyright (c) 2017, DCSO GmbH

import (
"encoding/json"
"net"
"sync"
"time"
Expand All @@ -25,6 +26,8 @@ type ForwardHandlerPerfStats struct {
// event types to be forwarded.
type ForwardHandler struct {
Logger *log.Entry
DoRDNS bool
RDNSHandler *RDNSHandler
ForwardEventChan chan []byte
OutputSocket string
OutputConn net.Conn
Expand Down Expand Up @@ -177,8 +180,24 @@ func MakeForwardHandler(reconnectTimes int, outputSocket string) *ForwardHandler
func (fh *ForwardHandler) Consume(e *types.Entry) error {
doForwardThis := util.ForwardAllEvents || util.AllowType(e.EventType)
if doForwardThis {
jsonCopy := make([]byte, len(e.JSONLine))
copy(jsonCopy, e.JSONLine)
var ev types.EveEvent
err := json.Unmarshal([]byte(e.JSONLine), &ev)
if err != nil {
return err
}
if fh.DoRDNS && fh.RDNSHandler != nil {
err = fh.RDNSHandler.Consume(e)
if err != nil {
return err
}
ev.SrcHost = e.SrcHosts
ev.DestHost = e.DestHosts
}
var jsonCopy []byte
jsonCopy, err = json.Marshal(ev)
if err != nil {
return err
}
fh.ForwardEventChan <- jsonCopy
fh.Lock.Lock()
fh.PerfStats.ForwardedPerSec++
Expand All @@ -201,6 +220,13 @@ func (fh *ForwardHandler) GetEventTypes() []string {
return util.GetAllowedTypes()
}

// EnableRDNS switches on reverse DNS enrichment for source and destination
// IPs in outgoing EVE events.
func (fh *ForwardHandler) EnableRDNS(expiryPeriod time.Duration) {
fh.DoRDNS = true
fh.RDNSHandler = MakeRDNSHandler(util.NewHostNamer(expiryPeriod, 2*expiryPeriod))
}

// Run starts forwarding of JSON representations of all consumed events
func (fh *ForwardHandler) Run() {
if !fh.Running {
Expand Down
110 changes: 110 additions & 0 deletions processing/rdns_handler.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,110 @@
package processing

// DCSO FEVER
// Copyright (c) 2019, DCSO GmbH

import (
"net"
"sync"

"github.com/DCSO/fever/types"
"github.com/DCSO/fever/util"

log "github.com/sirupsen/logrus"
"github.com/yl2chen/cidranger"
)

// RDNSHandler is a handler that enriches events with reverse DNS
// information looked up on the sensor, for both source and destination
// IP addresses.
type RDNSHandler struct {
sync.Mutex
Logger *log.Entry
HostNamer *util.HostNamer
PrivateRanges cidranger.Ranger
PrivateRangesOnly bool
}

// MakeRDNSHandler returns a new RDNSHandler, backed by the passed HostNamer.
func MakeRDNSHandler(hn *util.HostNamer) *RDNSHandler {
rh := &RDNSHandler{
Logger: log.WithFields(log.Fields{
"domain": "rdns",
}),
PrivateRanges: cidranger.NewPCTrieRanger(),
HostNamer: hn,
}
for _, cidr := range []string{
"10.0.0.0/8",
"172.16.0.0/12",
"192.168.0.0/16",
"fc00::/7",
} {
_, block, err := net.ParseCIDR(cidr)
if err != nil {
log.Fatalf("cannot parse fixed private IP range %v", cidr)
}
rh.PrivateRanges.Insert(cidranger.NewBasicRangerEntry(*block))
}
return rh
}

// EnableOnlyPrivateIPRanges ensures that only private (RFC1918) IP ranges
// are enriched
func (a *RDNSHandler) EnableOnlyPrivateIPRanges() {
a.PrivateRangesOnly = true
}

// Consume processes an Entry and enriches it
func (a *RDNSHandler) Consume(e *types.Entry) error {
var res []string
var err error
var isPrivate bool

if e.SrcIP != "" {
ip := net.ParseIP(e.SrcIP)
if ip != nil {
isPrivate, err = a.PrivateRanges.Contains(ip)
if err != nil {
return err
}
if !a.PrivateRangesOnly || isPrivate {
res, err = a.HostNamer.GetHostname(e.SrcIP)
if err == nil {
e.SrcHosts = res
}
}
} else {
log.Error("IP not valid")
}
}
if e.DestIP != "" {
ip := net.ParseIP(e.DestIP)
if ip != nil {
isPrivate, err = a.PrivateRanges.Contains(ip)
if err != nil {
return err
}
if !a.PrivateRangesOnly || isPrivate {
res, err = a.HostNamer.GetHostname(e.DestIP)
if err == nil {
e.DestHosts = res
}
}
} else {
log.Error("IP not valid")
}
}
return nil
}

// GetName returns the name of the handler
func (a *RDNSHandler) GetName() string {
return "reverse DNS handler"
}

// GetEventTypes returns a slice of event type strings that this handler
// should be applied to
func (a *RDNSHandler) GetEventTypes() []string {
return []string{"http", "dns", "tls", "smtp", "flow", "ssh", "tls", "smb", "alert"}
}
2 changes: 2 additions & 0 deletions types/entry.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,10 @@ type DNSAnswer struct {
// Entry is a collection of data that needs to be parsed FAST from the entry
type Entry struct {
SrcIP string
SrcHosts []string
SrcPort int64
DestIP string
DestHosts []string
DestPort int64
Timestamp string
EventType string
Expand Down
2 changes: 2 additions & 0 deletions types/eve.go
Original file line number Diff line number Diff line change
Expand Up @@ -241,8 +241,10 @@ type EveEvent struct {
InIface string `json:"in_iface,omitempty"`
SrcIP string `json:"src_ip,omitempty"`
SrcPort int `json:"src_port,omitempty"`
SrcHost []string `json:"src_host,omitempty"`
DestIP string `json:"dest_ip,omitempty"`
DestPort int `json:"dest_port,omitempty"`
DestHost []string `json:"dest_host,omitempty"`
Proto string `json:"proto,omitempty"`
AppProto string `json:"app_proto,omitempty"`
TxID int `json:"tx_id,omitempty"`
Expand Down
51 changes: 51 additions & 0 deletions util/hostnamer.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
package util

import (
"net"
"strings"
"sync"
"time"

"github.com/patrickmn/go-cache"
)

// HostNamer is a component that provides cached hostnames for IP
// addresses passed as strings.
type HostNamer struct {
Cache *cache.Cache
Lock sync.Mutex
}

// NewHostNamer returns a new HostNamer with the given default expiration time.
// Data entries will be purged after each cleanupInterval.
func NewHostNamer(defaultExpiration, cleanupInterval time.Duration) *HostNamer {
return &HostNamer{
Cache: cache.New(defaultExpiration, cleanupInterval),
}
}

// GetHostname returns a list of host names for a given IP address.
func (n *HostNamer) GetHostname(ipAddr string) ([]string, error) {
n.Lock.Lock()
defer n.Lock.Unlock()

val, found := n.Cache.Get(ipAddr)
if found {
return val.([]string), nil
}
hns, err := net.LookupAddr(ipAddr)
if err != nil {
return nil, err
}
for i, hn := range hns {
hns[i] = strings.TrimRight(hn, ".")
}
n.Cache.Set(ipAddr, hns, cache.DefaultExpiration)
val = hns
return val.([]string), nil
}

// Flush clears the cache of a HostNamer.
func (n *HostNamer) Flush() {
n.Cache.Flush()
}
Loading

0 comments on commit f82a9c5

Please # to comment.