Merge pull request #518 from jmpsec/zerolog-packages

Migrating native log to zerolog in osctrl-cli and other packages

carves/go.mod

@@ -31,6 +31,11 @@ require (
 	gorm.io/gorm v1.25.11
 )
 
+require (
+	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mattn/go-isatty v0.0.19 // indirect
+)
+
 require (
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.4 // indirect
 	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.13 // indirect
@@ -55,6 +60,7 @@ require (
 	github.com/magiconair/properties v1.8.7 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/pelletier/go-toml/v2 v2.2.3 // indirect
+	github.com/rs/zerolog v1.33.0
 	github.com/sagikazarmark/locafero v0.6.0 // indirect
 	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
 	github.com/segmentio/ksuid v1.0.4 // indirect

carves/go.sum

@@ -400,6 +400,7 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
 github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
 github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
+github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/denisenkom/go-mssqldb v0.0.0-20191124224453-732737034ffd/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU=
@@ -416,6 +417,7 @@ github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
+github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
@@ -502,6 +504,11 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY=
 github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
+github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
+github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
+github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
+github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA=
+github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/mattn/go-sqlite3 v1.14.0/go.mod h1:JIl7NbARA7phWnGvh0LKTyg7S9BA+6gx71ShQilpsus=
 github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
 github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
@@ -520,6 +527,9 @@ github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qR
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
+github.com/rs/zerolog v1.33.0 h1:1cU2KZkvPxNyfgEmhHAz/1A9Bz+llsdYzklWFzgp0r8=
+github.com/rs/zerolog v1.33.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss=
 github.com/sagikazarmark/locafero v0.3.0 h1:zT7VEGWC2DTflmccN/5T1etyKvxSxpHsjb9cJvm4SvQ=
 github.com/sagikazarmark/locafero v0.3.0/go.mod h1:w+v7UsPNFwzF1cHuOajOOzoq4U7v/ig1mpRjqV+Bu1U=
 github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ=
@@ -725,6 +735,9 @@ golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q=
 golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=

carves/s3.go

@@ -6,13 +6,13 @@ import (
 	"encoding/base64"
 	"fmt"
 	"io"
-	"log"
 	"net/http"
 	"net/url"
 	"time"
 
 	"github.com/jmpsec/osctrl/settings"
 	"github.com/jmpsec/osctrl/types"
+	"github.com/rs/zerolog/log"
 	"github.com/spf13/viper"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
@@ -78,7 +78,7 @@ func CreateCarverS3(s3Config types.S3Configuration) (*CarverS3, error) {
 // LoadS3 - Function to load the S3 configuration from JSON file
 func LoadS3(file string) (types.S3Configuration, error) {
 	var _s3Cfg types.S3Configuration
-	log.Printf("Loading %s", file)
+	log.Info().Msgf("Loading %s", file)
 	// Load file and read config
 	viper.SetConfigFile(file)
 	if err := viper.ReadInConfig(); err != nil {
@@ -94,14 +94,14 @@ func LoadS3(file string) (types.S3Configuration, error) {
 
 // Settings - Function to prepare settings for the logger
 func (carveS3 *CarverS3) Settings(mgr *settings.Settings) {
-	log.Printf("No s3 logging settings\n")
+	log.Info().Msg("No s3 logging settings")
 }
 
 // Upload - Function that sends data from carves to S3
 func (carveS3 *CarverS3) Upload(block CarvedBlock, uuid, data string) error {
 	ctx := context.Background()
 	if carveS3.Debug {
-		log.Printf("DebugService: Sending %d bytes to S3 for %s - %s", block.Size, block.Environment, uuid)
+		log.Debug().Msgf("DebugService: Sending %d bytes to S3 for %s - %s", block.Size, block.Environment, uuid)
 	}
 	// Decode before upload
 	toUpload, err := base64.StdEncoding.DecodeString(data)
@@ -120,7 +120,7 @@ func (carveS3 *CarverS3) Upload(block CarvedBlock, uuid, data string) error {
 		return fmt.Errorf("error sending data to s3 - %s", err)
 	}
 	if carveS3.Debug {
-		log.Printf("DebugService: S3 Upload %+v", uploadOutput)
+		log.Debug().Msgf("DebugService: S3 Upload %+v", uploadOutput)
 	}
 	return nil
 }
@@ -192,7 +192,7 @@ func (carveS3 *CarverS3) Archive(carve CarvedFile, blocks []CarvedBlock) (*Carve
 		return nil, fmt.Errorf("CompleteMultipartUpload - %s", err)
 	}
 	if carveS3.Debug {
-		log.Printf("DebugService: S3 Archived %s [%d bytes] - %s", res.File, res.Size, *multiOutput.Key)
+		log.Debug().Msgf("DebugService: S3 Archived %s [%d bytes] - %s", res.File, res.Size, *multiOutput.Key)
 	}
 	return res, nil
 }
@@ -201,7 +201,7 @@ func (carveS3 *CarverS3) Archive(carve CarvedFile, blocks []CarvedBlock) (*Carve
 func (carveS3 *CarverS3) Download(carve CarvedFile) (io.WriterAt, error) {
 	ctx := context.Background()
 	if carveS3.Debug {
-		log.Printf("DebugService: Downloading %s from S3", carve.ArchivePath)
+		log.Debug().Msgf("DebugService: Downloading %s from S3", carve.ArchivePath)
 	}
 	downloader := manager.NewDownloader(carveS3.Client)
 	var fileReader io.WriterAt
@@ -215,7 +215,7 @@ func (carveS3 *CarverS3) Download(carve CarvedFile) (io.WriterAt, error) {
 		return nil, fmt.Errorf("Download - %s", err)
 	}
 	if carveS3.Debug {
-		log.Printf("DebugService: S3 Downloaded %s [%d bytes]", carve.ArchivePath, downloadedBytes)
+		log.Debug().Msgf("DebugService: S3 Downloaded %s [%d bytes]", carve.ArchivePath, downloadedBytes)
 	}
 	return fileReader, nil
 }
@@ -224,7 +224,7 @@ func (carveS3 *CarverS3) Download(carve CarvedFile) (io.WriterAt, error) {
 func (carveS3 *CarverS3) GetDownloadLink(carve CarvedFile) (string, error) {
 	ctx := context.Background()
 	if carveS3.Debug {
-		log.Printf("DebugService: Downloading link %s from S3", carve.ArchivePath)
+		log.Debug().Msgf("DebugService: Downloading link %s from S3", carve.ArchivePath)
 	}
 	preClient := s3.NewPresignClient(carveS3.Client)
 	lnk, err := preClient.PresignGetObject(ctx, &s3.GetObjectInput{

cli/api-carve.go

@@ -3,11 +3,11 @@ package main
 import (
 	"encoding/json"
 	"fmt"
-	"log"
 	"strings"
 
 	"github.com/jmpsec/osctrl/carves"
 	"github.com/jmpsec/osctrl/types"
+	"github.com/rs/zerolog/log"
 )
 
 // GetCarves to retrieve carves from osctrl
@@ -58,7 +58,7 @@ func (api *OsctrlAPI) RunCarve(env, uuid, path string) (types.ApiQueriesResponse
 	reqURL := fmt.Sprintf("%s%s%s/%s", api.Configuration.URL, APIPath, APICarves, env)
 	jsonMessage, err := json.Marshal(c)
 	if err != nil {
-		log.Printf("error marshaling data %s", err)
+		log.Err(err).Msg("error marshaling data")
 	}
 	jsonParam := strings.NewReader(string(jsonMessage))
 	rawC, err := api.PostGeneric(reqURL, jsonParam)

cli/api-node.go

@@ -3,7 +3,6 @@ package main
 import (
 	"encoding/json"
 	"fmt"
-	"log"
 	"strings"
 
 	"github.com/jmpsec/osctrl/nodes"
@@ -47,7 +46,7 @@ func (api *OsctrlAPI) DeleteNode(env, identifier string) error {
 	reqURL := fmt.Sprintf("%s%s%s/%s/delete", api.Configuration.URL, APIPath, APINodes, env)
 	jsonMessage, err := json.Marshal(n)
 	if err != nil {
-		log.Printf("error marshaling data %s", err)
+		return fmt.Errorf("error marshaling data - %v", err)
 	}
 	jsonParam := strings.NewReader(string(jsonMessage))
 	rawN, err := api.PostGeneric(reqURL, jsonParam)

cli/api-query.go

@@ -3,7 +3,6 @@ package main
 import (
 	"encoding/json"
 	"fmt"
-	"log"
 	"strings"
 
 	"github.com/jmpsec/osctrl/queries"
@@ -51,15 +50,16 @@ func (api *OsctrlAPI) CompleteQuery(env, identifier string) error {
 // RunQuery to initiate a query in osctrl
 func (api *OsctrlAPI) RunQuery(env, uuid, query string, hidden bool) (types.ApiQueriesResponse, error) {
 	q := types.ApiDistributedQueryRequest{
-		UUIDs:   []string{uuid},
+		UUIDs:  []string{uuid},
 		Query:  query,
 		Hidden: hidden,
 	}
 	var r types.ApiQueriesResponse
 	reqURL := fmt.Sprintf("%s%s%s/%s", api.Configuration.URL, APIPath, APIQueries, env)
 	jsonMessage, err := json.Marshal(q)
 	if err != nil {
-		log.Printf("error marshaling data %s", err)
+		return r, fmt.Errorf("error marshaling data - %v", err)
+
 	}
 	jsonParam := strings.NewReader(string(jsonMessage))
 	rawQ, err := api.PostGeneric(reqURL, jsonParam)

cli/api.go

@@ -6,10 +6,10 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"log"
 	"net/http"
 	"net/url"
+	"os"
 
 	"github.com/jmpsec/osctrl/version"
 	"github.com/spf13/viper"
@@ -88,7 +88,7 @@ func writeAPIConfiguration(file string, apiConf JSONConfigurationAPI) error {
 	if err != nil {
 		return fmt.Errorf("error serializing data %s", err)
 	}
-	if err := ioutil.WriteFile(file, confByte, 0644); err != nil {
+	if err := os.WriteFile(file, confByte, 0644); err != nil {
 		return fmt.Errorf("error writing to file %s", err)
 	}
 	return nil
@@ -154,12 +154,7 @@ func (api *OsctrlAPI) ReqGeneric(reqType string, url string, body io.Reader) ([]
 	if err != nil {
 		return []byte{}, fmt.Errorf("Client.Do - %v", err)
 	}
-	//defer resp.Body.Close()
-	defer func() {
-		if err := resp.Body.Close(); err != nil {
-			log.Printf("failed to close body %v", err)
-		}
-	}()
+	defer resp.Body.Close()
 	// Read body
 	bodyBytes, err := io.ReadAll(resp.Body)
 	if err != nil {

logging/db.go

@@ -3,7 +3,6 @@ package logging
 import (
 	"encoding/json"
 	"fmt"
-	"log"
 	"strconv"
 	"strings"
 	"time"
@@ -14,6 +13,7 @@ import (
 	"github.com/jmpsec/osctrl/queries"
 	"github.com/jmpsec/osctrl/settings"
 	"github.com/jmpsec/osctrl/types"
+	"github.com/rs/zerolog/log"
 )
 
 const (
@@ -89,28 +89,28 @@ func CreateLoggerDB(backend *backend.DBManager) (*LoggerDB, error) {
 	}
 	// table osquery_status_data
 	if err := backend.Conn.AutoMigrate(&OsqueryStatusData{}); err != nil {
-		log.Fatalf("Failed to AutoMigrate table (osquery_status_data): %v", err)
+		log.Fatal().Msgf("Failed to AutoMigrate table (osquery_status_data): %v", err)
 	}
 	// table osquery_result_data
 	if err := backend.Conn.AutoMigrate(&OsqueryResultData{}); err != nil {
-		log.Fatalf("Failed to AutoMigrate table (osquery_result_data): %v", err)
+		log.Fatal().Msgf("Failed to AutoMigrate table (osquery_result_data): %v", err)
 	}
 	// table osquery_query_data
 	if err := backend.Conn.AutoMigrate(&OsqueryQueryData{}); err != nil {
-		log.Fatalf("Failed to AutoMigrate table (osquery_query_data): %v", err)
+		log.Fatal().Msgf("Failed to AutoMigrate table (osquery_query_data): %v", err)
 	}
 	return l, nil
 }
 
 // Settings - Function to prepare settings for the logger
 func (logDB *LoggerDB) Settings(mgr *settings.Settings) {
-	//log.Printf("Setting DB logging settings\n")
+	log.Info().Msg("Setting DB logging settings")
 }
 
 // Log - Function that sends JSON result/status/query logs to the configured DB
 func (logDB *LoggerDB) Log(logType string, data []byte, environment, uuid string, debug bool) {
 	if debug {
-		log.Printf("Sending %d bytes to DB for %s - %s", len(data), environment, uuid)
+		log.Debug().Msgf("Sending %d bytes to DB for %s - %s", len(data), environment, uuid)
 	}
 	switch logType {
 	case types.StatusLog:
@@ -125,7 +125,7 @@ func (logDB *LoggerDB) Status(data []byte, environment, uuid string, debug bool)
 	// Parse JSON
 	var logs []types.LogStatusData
 	if err := json.Unmarshal(data, &logs); err != nil {
-		log.Printf("error parsing logs %s %v", string(data), err)
+		log.Err(err).Msgf("error parsing logs %s %v", string(data), err)
 	}
 	// Iterate and insert in DB
 	for _, l := range logs {
@@ -139,7 +139,7 @@ func (logDB *LoggerDB) Status(data []byte, environment, uuid string, debug bool)
 			Severity:    strconv.Itoa(int(l.Severity)),
 		}
 		if err := logDB.Database.Conn.Create(&entry).Error; err != nil {
-			log.Printf("Error creating status log entry %s", err)
+			log.Err(err).Msg("Error creating status log entry")
 		}
 	}
 }
@@ -149,7 +149,7 @@ func (logDB *LoggerDB) Result(data []byte, environment, uuid string, debug bool)
 	// Parse JSON
 	var logs []types.LogResultData
 	if err := json.Unmarshal(data, &logs); err != nil {
-		log.Printf("error parsing logs %s %v", string(data), err)
+		log.Err(err).Msgf("error parsing logs %s", string(data))
 	}
 	// Iterate and insert in DB
 	for _, l := range logs {
@@ -163,7 +163,7 @@ func (logDB *LoggerDB) Result(data []byte, environment, uuid string, debug bool)
 			Counter:     l.Counter,
 		}
 		if err := logDB.Database.Conn.Create(&entry).Error; err != nil {
-			log.Printf("Error creating result log entry %s", err)
+			log.Err(err).Msg("Error creating result log entry")
 		}
 	}
 }
@@ -180,7 +180,7 @@ func (logDB *LoggerDB) Query(data []byte, environment, uuid, name string, status
 	}
 	// Insert in DB
 	if err := logDB.Database.Conn.Create(&entry).Error; err != nil {
-		log.Printf("Error creating query log %s", err)
+		log.Err(err).Msg("Error creating query log")
 	}
 }