Skip to content

Commit

Permalink
12 nonsense values in pv generation today metric (#20)
Browse files Browse the repository at this point in the history
  • Loading branch information
kubaceg authored Feb 5, 2025
1 parent 4c9737f commit 1fe7f55
Show file tree
Hide file tree
Showing 15 changed files with 238 additions and 69 deletions.
3 changes: 3 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -8,3 +8,6 @@ build:

test:
go test -v ./...

lint:
docker run -t --rm -v .:/app -v ~/.cache/golangci-lint/v1.63.4:/root/.cache -w /app golangci/golangci-lint:v1.63.4 golangci-lint run -v
20 changes: 10 additions & 10 deletions adapters/devices/sofar/device.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,15 @@ type Logger struct {
attrBlackList []*regexp.Regexp
}

func NewSofarLogger(serialNumber uint, connPort ports.CommunicationPort, attrWhiteList []string, attrBlackList []string) *Logger {
return &Logger{
serialNumber: serialNumber,
connPort: connPort,
attrWhiteList: toSet(attrWhiteList),
attrBlackList: toREs(attrBlackList),
}
}

// for a set in go we use a map of keys -> empty struct
func toSet(slice []string) map[string]struct{} {
set := make(map[string]struct{}, len(slice))
Expand All @@ -38,15 +47,6 @@ func toREs(patterns []string) []*regexp.Regexp {
return res
}

func NewSofarLogger(serialNumber uint, connPort ports.CommunicationPort, attrWhiteList []string, attrBlackList []string) *Logger {
return &Logger{
serialNumber: serialNumber,
connPort: connPort,
attrWhiteList: toSet(attrWhiteList),
attrBlackList: toREs(attrBlackList),
}
}

func (s *Logger) nameFilter(k string) bool {
if len(s.attrWhiteList) > 0 {
if _, ok := s.attrWhiteList[k]; ok {
Expand All @@ -65,7 +65,7 @@ func (s *Logger) GetDiscoveryFields() []ports.DiscoveryField {
return getDiscoveryFields(s.nameFilter)
}

func (s *Logger) Query() (map[string]interface{}, error) {
func (s *Logger) Query() (ports.MeasurementMap, error) {
return readData(s.connPort, s.serialNumber, s.nameFilter)
}

Expand Down
7 changes: 3 additions & 4 deletions adapters/devices/sofar/lsw.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,6 @@ func (l LSWRequest) ToBytes() []byte {
buf[5] = 0x00
buf[6] = 0x00

// fmt.Printf("serial number: %0X\n", uint32SerialNumber)
binary.LittleEndian.PutUint32(buf[7:], uint32(l.serialNumber))

buf[11] = 0x02
Expand Down Expand Up @@ -73,8 +72,8 @@ func (l LSWRequest) checksum(buf []byte) uint8 {
return checksum
}

func readData(connPort ports.CommunicationPort, serialNumber uint, nameFilter func(string) bool) (map[string]interface{}, error) {
result := make(map[string]interface{})
func readData(connPort ports.CommunicationPort, serialNumber uint, nameFilter func(string) bool) (ports.MeasurementMap, error) {
result := make(ports.MeasurementMap)

for _, rr := range allRegisterRanges {
reply, err := readRegisterRange(rr, connPort, serialNumber)
Expand All @@ -90,7 +89,7 @@ func readData(connPort ports.CommunicationPort, serialNumber uint, nameFilter fu
return result, nil
}

func readRegisterRange(rr registerRange, connPort ports.CommunicationPort, serialNumber uint) (map[string]interface{}, error) {
func readRegisterRange(rr registerRange, connPort ports.CommunicationPort, serialNumber uint) (ports.MeasurementMap, error) {
lswRequest := NewLSWRequest(serialNumber, rr.start, rr.end)

commandBytes := lswRequest.ToBytes()
Expand Down
2 changes: 1 addition & 1 deletion adapters/export/mosquitto/mqtt.go
Original file line number Diff line number Diff line change
Expand Up @@ -126,7 +126,7 @@ func (conn *Connection) InsertDiscoveryRecord(discovery string, state string, fi
return nil
}

func (conn *Connection) InsertRecord(m map[string]interface{}) error {
func (conn *Connection) InsertRecord(m ports.MeasurementMap) error {
json, _ := json.Marshal(m)
conn.publish(conn.prefix, string(json), false) // state messages should not be retained
return nil
Expand Down
3 changes: 2 additions & 1 deletion adapters/export/otlp/metrics.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"log/slog"

"github.com/kubaceg/sofar_g3_lsw3_logger_reader/adapters/devices/sofar"
"github.com/kubaceg/sofar_g3_lsw3_logger_reader/ports"
grpc "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc"
http "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp"
"go.opentelemetry.io/otel/metric"
Expand Down Expand Up @@ -119,7 +120,7 @@ func (s *Service) createGauge(n string) *instrument.Int64ObservableGauge {
}

// CollectAndPushMetrics triggers the collection and export of metrics over OTLP
func (s *Service) CollectAndPushMetrics(ctx context.Context, measurements map[string]interface{}) error {
func (s *Service) CollectAndPushMetrics(ctx context.Context, measurements ports.MeasurementMap) error {
s.measurements = measurements
rm := metricdata.ResourceMetrics{}
if err := s.reader.Collect(ctx, &rm); err != nil {
Expand Down
36 changes: 36 additions & 0 deletions adapters/filters/daily_generation.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
package filters

import (
"errors"

"github.com/kubaceg/sofar_g3_lsw3_logger_reader/ports"
)

var ErrDailyGenerationDiffTooHigh = errors.New("daily generation spike detected, skipping")

type DailyGenerationSpikesFilter struct {
lastDailyGenerationValue uint32
maxDailyGenerationDiff uint32
}

func NewDailyGenerationFilter(maxDailyGenerationDiff uint32) *DailyGenerationSpikesFilter {
return &DailyGenerationSpikesFilter{
maxDailyGenerationDiff: maxDailyGenerationDiff,
}
}

func (d *DailyGenerationSpikesFilter) Filter(data ports.MeasurementMap) (ports.MeasurementMap, error) {
// If the last daily generation value is higher than the current one, it means a new day has started
if data["PV_Generation_Today"].(uint32) < d.lastDailyGenerationValue {
d.lastDailyGenerationValue = 0
}

if d.lastDailyGenerationValue > 0 &&
data["PV_Generation_Today"].(uint32)-d.lastDailyGenerationValue > d.maxDailyGenerationDiff {
return nil, ErrDailyGenerationDiffTooHigh
}

d.lastDailyGenerationValue = data["PV_Generation_Today"].(uint32)

return data, nil
}
72 changes: 72 additions & 0 deletions adapters/filters/daily_generation_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
package filters

import (
"testing"

"github.com/kubaceg/sofar_g3_lsw3_logger_reader/ports"
"github.com/stretchr/testify/assert"
)

func TestFilterCases(t *testing.T) {
tests := []struct {
name string
lastDailyGenerationValue uint32
data ports.MeasurementMap
expectedResult ports.MeasurementMap
expectedError error
}{
{
name: "ValidData",
lastDailyGenerationValue: uint32(0),
data: ports.MeasurementMap{"PV_Generation_Today": uint32(5000)},
expectedResult: ports.MeasurementMap{"PV_Generation_Today": uint32(5000)},
expectedError: nil,
},
{
name: "DiffTooHigh",
lastDailyGenerationValue: uint32(9000),
data: ports.MeasurementMap{"PV_Generation_Today": uint32(20000)},
expectedResult: nil,
expectedError: ErrDailyGenerationDiffTooHigh,
},
{
name: "FirstDataPoint",
lastDailyGenerationValue: uint32(0),
data: ports.MeasurementMap{"PV_Generation_Today": uint32(15000)},
expectedResult: ports.MeasurementMap{"PV_Generation_Today": uint32(15000)},
expectedError: nil,
},
{
name: "ExactMaxDiff",
lastDailyGenerationValue: uint32(10000),
data: ports.MeasurementMap{"PV_Generation_Today": uint32(20000)},
expectedResult: ports.MeasurementMap{"PV_Generation_Today": uint32(20000)},
expectedError: nil,
},
{
name: "NewDay",
lastDailyGenerationValue: uint32(10000),
data: ports.MeasurementMap{"PV_Generation_Today": uint32(500)},
expectedResult: ports.MeasurementMap{"PV_Generation_Today": uint32(500)},
expectedError: nil,
},
}

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
filter := NewDailyGenerationFilter(10000)
filter.lastDailyGenerationValue = tt.lastDailyGenerationValue

result, err := filter.Filter(tt.data)

if tt.expectedError != nil {
assert.Error(t, err)
assert.Equal(t, tt.expectedError.Error(), err.Error())
} else {
assert.NoError(t, err)
}

assert.Equal(t, tt.expectedResult, result)
})
}
}
4 changes: 4 additions & 0 deletions config-example.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,10 @@ inverter:
- "^[ST]_" # prefix R_, S_, T_ for 3 phases, only R_ used in single phase systems
- "_[ST]$" # likewise suffixes _R, _S, _T

filters:
dailyGenerationSpikes: 100000 # daily generation spikes (difference between last value and current readed value) above this value (in wats) are ignored,
# here is issue which was solved by this filter https://github.com/kubaceg/Sofar-g3-lsw3-logger-reader/issues/12

mqtt: # MQTT disabled if url & prefix both blank
url: 1.2.3.4:1883 # MQTT broker URL (e.g. 1.2.3.4:1883)
user: # MQTT username (leave empty when not needed)
Expand Down
7 changes: 5 additions & 2 deletions config.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,11 @@ type Config struct {
AttrWhiteList []string `yaml:"attrWhiteList"`
AttrBlackList []string `yaml:"attrBlackList"`
} `yaml:"inverter"`
Mqtt mosquitto.MqttConfig `yaml:"mqtt"`
Otlp otlp.Config `yaml:"otlp"`
Mqtt mosquitto.MqttConfig `yaml:"mqtt"`
Otlp otlp.Config `yaml:"otlp"`
Filters struct {
DailyGenerationSpikes uint32 `default:"0" yaml:"dailyGenerationSpikes"`
} `yaml:"filters"`
}

func (c *Config) validate() error {
Expand Down
4 changes: 4 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ go 1.21
require (
github.com/eclipse/paho.mqtt.golang v1.3.1
github.com/sigurn/crc16 v0.0.0-20211026045750-20ab5afb07e3
github.com/stretchr/testify v1.8.2
go.bug.st/serial v1.4.0
go.opentelemetry.io/otel v1.14.0
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.37.0
Expand All @@ -18,11 +19,13 @@ require (
require (
github.com/cenkalti/backoff/v4 v4.2.0 // indirect
github.com/creack/goselect v0.1.2 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/go-logr/logr v1.2.3 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/gorilla/websocket v1.4.2 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.14.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.37.0 // indirect
go.opentelemetry.io/otel/trace v1.14.0 // indirect
Expand All @@ -35,4 +38,5 @@ require (
google.golang.org/genproto/googleapis/rpc v0.0.0-20231012201019-e917dd12ba7a // indirect
google.golang.org/grpc v1.58.2 // indirect
google.golang.org/protobuf v1.31.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)
4 changes: 4 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -157,9 +157,13 @@ github.com/sigurn/crc16 v0.0.0-20211026045750-20ab5afb07e3 h1:aQKxg3+2p+IFXXg97M
github.com/sigurn/crc16 v0.0.0-20211026045750-20ab5afb07e3/go.mod h1:9/etS5gpQq9BJsJMWg1wpLbfuSnkm8dPF6FdW2JXVhA=
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
Expand Down
Loading

0 comments on commit 1fe7f55

Please # to comment.