From 39e718772232f639dc14537f401b45a38e469387 Mon Sep 17 00:00:00 2001
From: Neil Bacon <2782001+neilbacon@users.noreply.github.com>
Date: Sun, 20 Aug 2023 15:23:22 +1000
Subject: [PATCH 01/16] add attribute white list

---
 adapters/devices/sofar/device.go  | 26 ++++++++++++----
 adapters/devices/sofar/lsw.go     | 12 ++++++--
 adapters/export/mosquitto/mqtt.go | 50 ++++++++++++++++++++-----------
 config-example.yaml               | 38 ++++++++++++++++-------
 config.go                         | 11 ++++---
 go.sum                            |  4 +++
 main.go                           |  2 +-
 7 files changed, 100 insertions(+), 43 deletions(-)

diff --git a/adapters/devices/sofar/device.go b/adapters/devices/sofar/device.go
index 2671919..1ec1410 100644
--- a/adapters/devices/sofar/device.go
+++ b/adapters/devices/sofar/device.go
@@ -3,19 +3,33 @@ package sofar
 import "github.com/kubaceg/sofar_g3_lsw3_logger_reader/ports"
 
 type Logger struct {
-	serialNumber uint
-	connPort     ports.CommunicationPort
+	serialNumber  uint
+	connPort      ports.CommunicationPort
+	attrWhiteList map[string]struct{}
+	attrBlackList []string
 }
 
-func NewSofarLogger(serialNumber uint, connPort ports.CommunicationPort) *Logger {
+// for a set in go we use a map of keys -> empty struct
+func toSet(slice []string) map[string]struct{} {
+	set := make(map[string]struct{}, len(slice))
+	v := struct{}{}
+	for _, s := range slice {
+		set[s] = v
+	}
+	return set
+}
+
+func NewSofarLogger(serialNumber uint, connPort ports.CommunicationPort, attrWhiteList []string, attrBlackList []string) *Logger {
 	return &Logger{
-		serialNumber: serialNumber,
-		connPort:     connPort,
+		serialNumber:  serialNumber,
+		connPort:      connPort,
+		attrWhiteList: toSet(attrWhiteList),
+		attrBlackList: attrBlackList,
 	}
 }
 
 func (s *Logger) Query() (map[string]interface{}, error) {
-	return readData(s.connPort, s.serialNumber)
+	return readData(s.connPort, s.serialNumber, s.attrWhiteList, s.attrBlackList)
 }
 
 func (s *Logger) Name() string {
diff --git a/adapters/devices/sofar/lsw.go b/adapters/devices/sofar/lsw.go
index b8246a1..f1c1a9f 100644
--- a/adapters/devices/sofar/lsw.go
+++ b/adapters/devices/sofar/lsw.go
@@ -73,7 +73,7 @@ func (l LSWRequest) checksum(buf []byte) uint8 {
 	return checksum
 }
 
-func readData(connPort ports.CommunicationPort, serialNumber uint) (map[string]interface{}, error) {
+func readData(connPort ports.CommunicationPort, serialNumber uint, attrWhiteList map[string]struct{}, attrBlackList []string) (map[string]interface{}, error) {
 	result := make(map[string]interface{})
 
 	for _, rr := range allRegisterRanges {
@@ -81,9 +81,15 @@ func readData(connPort ports.CommunicationPort, serialNumber uint) (map[string]i
 		if err != nil {
 			return nil, err
 		}
-
 		for k, v := range reply {
-			result[k] = v
+			ok := len(attrWhiteList) == 0
+			if !ok { // TODO: also handle attrBlackList
+				_, ok = attrWhiteList[k]
+			}
+			// log.Printf("readData: %s %v", k, ok)
+			if ok {
+				result[k] = v
+			}
 		}
 	}
 	return result, nil
diff --git a/adapters/export/mosquitto/mqtt.go b/adapters/export/mosquitto/mqtt.go
index 80305ce..8a7a4fd 100644
--- a/adapters/export/mosquitto/mqtt.go
+++ b/adapters/export/mosquitto/mqtt.go
@@ -55,27 +55,41 @@ func New(config *MqttConfig) (*Connection, error) {
 
 }
 
+func publish(conn *Connection, k string, v interface{}) {
+	token := conn.client.Publish(fmt.Sprintf("%s/%s", conn.prefix, k), 0, true, fmt.Sprintf("%v", v))
+	res := token.WaitTimeout(1 * time.Second)
+	if !res || token.Error() != nil {
+		log.Printf("error inserting to MQTT: %s", token.Error())
+	}
+}
+
+// next thing to do is add discovery
+// func discovery() {
+// MQTT Discovery: https://www.home-assistant.io/integrations/mqtt/#mqtt-discovery
+
+// homeassistant/sensor/inverter/config
+// payload
+// unique_id: PV_Generation_Today01ad
+// state_topic: "homeassistant/sensor/inverter/PV_Generation_Today"
+// device_class: energy
+// state_class: measurement
+// unit_of_measurement: 'kWh'
+// value_template: "{{ value|int * 0.01 }}"```
+
+// {"name": null, "device_class": "motion", "state_topic": "homeassistant/binary_sensor/garden/state", "unique_id": "motion01ad", "device": {"identifiers": ["01ad"], "name": "Garden" }}
+//}
+
 func (conn *Connection) InsertRecord(measurement map[string]interface{}) error {
-	measurementCopy := make(map[string]interface{}, len(measurement))
+	m := make(map[string]interface{}, len(measurement))
 	for k, v := range measurement {
-		measurementCopy[k] = v
+		m[k] = v
+	}
+	m["LastTimestamp"] = time.Now().UnixNano() / int64(time.Millisecond)
+	all, _ := json.Marshal(m)
+	m["All"] = string(all)
+	for k, v := range m {
+		publish(conn, k, v)
 	}
-	go func(measurement map[string]interface{}) {
-		// timestamp it
-		measurement["LastTimestamp"] = time.Now().UnixNano() / int64(time.Millisecond)
-		m, _ := json.Marshal(measurement)
-		measurement["All"] = string(m)
-
-		for k, v := range measurement {
-			token := conn.client.Publish(fmt.Sprintf("%s/%s", conn.prefix, k), 0, true, fmt.Sprintf("%v", v))
-			res := token.WaitTimeout(1 * time.Second)
-			if !res || token.Error() != nil {
-				log.Printf("error inserting to MQTT: %s", token.Error())
-			}
-		}
-
-	}(measurementCopy)
-
 	return nil
 }
 
diff --git a/config-example.yaml b/config-example.yaml
index 1ac32e9..064b326 100644
--- a/config-example.yaml
+++ b/config-example.yaml
@@ -1,15 +1,31 @@
+# make your own in config.yaml
+
 inverter:
-  port: 1.2.3.4:23 # required port name (e.g. /dev/ttyUSB0 for serial or 1.2.3.4:23 for TCP/IP
-  loggerSerial: 123456 # required logger serial number
-  readInterval: 60 # update interval in seconds, default 60
-mqtt:
-  url: 1.2.3.4:5678 #MQTT broker URL (e.g. 1.2.3.4:5678)
-  user: #MQTT username (leave empty when not needed)
-  password: #MQTT password (leave empty when not needed)
-  prefix: /sensors/energy/inverter #topic prefix on which data will be sent
-otlp:
+  port: 1.2.3.4:8899      # required, port name e.g. /dev/ttyUSB0 for serial or 1.2.3.4:8899 for TCP/IP
+  loggerSerial: 23XXXXXXX # required, logger serial number
+  readInterval: 60        # update interval in seconds, default 60
+  #
+  # if there is a non-empty attrWhiteList then only these explicitly listed attributes are output
+  attrWhiteList:
+    - ActivePower_Output_Total # Total PV generation (10W)
+    - Power_PV1                # PV output of string 1 (10W)
+    - Power_PV2                # PV output of string 2 (10W); PV1 + PV2 > ActivePower_Output_Total by about 3.5% due to inverter inefficiency?
+    - ActivePower_Load_Sys     # total power consumption (10W)
+    - ActivePower_PCC_Total    # grid export (+) or import (-) (10W) = ActivePower_Output_Total - ActivePower_Load_Sys (so it is redundant data)
+  # else attributes are output unless they match a regex in attrBlackList
+  attrBlackList:
+    - "^[ST]_"                 # prefix R_, S_, T_ for 3 phases, only R_ used in single phase systems
+    - "_[ST]$"                 # likewise suffixes _R, _S, _T
+
+mqtt:                     # MQTT disabled if url & prefix both blank
+  url: 1.2.3.4:1883       # MQTT broker URL (e.g. 1.2.3.4:1883)
+  user:                   # MQTT username (leave empty when not needed)
+  password:               # MQTT password (leave empty when not needed)
+  prefix: /sensors/energy/inverter # topic prefix on which data will be sent
+
+otlp:                     # OTLP disabled if both urls blank
   grpc:
-    url: 0.0.0.0:4317 #URL for gRPC OTLP server
+    url:                  # URL for gRPC OTLP server e.g. 0.0.0.0:4317
   http:
-    url: 0.0.0.0:4318 #URL for HTTP OLTP server
+    url:                  # URL for HTTP OLTP server e.g. 0.0.0.0:4318 
   prefix: sofar.logging
diff --git a/config.go b/config.go
index 424e399..be6ad9e 100644
--- a/config.go
+++ b/config.go
@@ -2,18 +2,21 @@ package main
 
 import (
 	"errors"
-	"github.com/kubaceg/sofar_g3_lsw3_logger_reader/adapters/export/otlp"
 	"os"
 
+	"github.com/kubaceg/sofar_g3_lsw3_logger_reader/adapters/export/otlp"
+
 	"github.com/kubaceg/sofar_g3_lsw3_logger_reader/adapters/export/mosquitto"
 	"gopkg.in/yaml.v2"
 )
 
 type Config struct {
 	Inverter struct {
-		Port         string `yaml:"port"`
-		LoggerSerial uint   `yaml:"loggerSerial"`
-		ReadInterval int    `default:"60" yaml:"readInterval"`
+		Port          string   `yaml:"port"`
+		LoggerSerial  uint     `yaml:"loggerSerial"`
+		ReadInterval  int      `default:"60" yaml:"readInterval"`
+		AttrWhiteList []string `yaml:"attrWhiteList"`
+		AttrBlackList []string `yaml:"attrBlackList"`
 	} `yaml:"inverter"`
 	Mqtt mosquitto.MqttConfig `yaml:"mqtt"`
 	Otlp otlp.Config          `yaml:"otlp"`
diff --git a/go.sum b/go.sum
index 53c4569..7fab8e7 100644
--- a/go.sum
+++ b/go.sum
@@ -203,6 +203,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0
 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
 golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
+golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63 h1:m64FZMko/V45gv0bNmrNYoDEq8U5YUhetc9cBWKS1TQ=
+golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63/go.mod h1:0v4NqG35kSWCMzLaMeX+IQrlSnVE/bqGSyC2cz/9Le8=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
 golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@@ -297,6 +299,8 @@ golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM=
+golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
diff --git a/main.go b/main.go
index c475640..0d8c9f0 100644
--- a/main.go
+++ b/main.go
@@ -66,7 +66,7 @@ func initialize() {
 		}
 	}
 
-	device = sofar.NewSofarLogger(config.Inverter.LoggerSerial, port)
+	device = sofar.NewSofarLogger(config.Inverter.LoggerSerial, port, config.Inverter.AttrWhiteList, config.Inverter.AttrBlackList)
 }
 
 func main() {

From d25f93de95fb61d1742a66eac72f3438d51bd5c3 Mon Sep 17 00:00:00 2001
From: Neil Bacon <2782001+neilbacon@users.noreply.github.com>
Date: Mon, 28 Aug 2023 18:27:41 +1000
Subject: [PATCH 02/16] discovery

---
 adapters/devices/sofar/device.go         | 14 ++++-
 adapters/devices/sofar/lsw.go            |  9 +--
 adapters/devices/sofar/sofar_protocol.go | 16 +++++
 adapters/export/mosquitto/mqtt.go        | 74 +++++++++++++++---------
 config-example.yaml                      |  6 +-
 main.go                                  | 41 ++++++-------
 ports/database.go                        |  1 +
 ports/devices.go                         |  8 +++
 sh/cleanmqtt.sh                          | 13 +++++
 9 files changed, 122 insertions(+), 60 deletions(-)
 create mode 100644 sh/cleanmqtt.sh

diff --git a/adapters/devices/sofar/device.go b/adapters/devices/sofar/device.go
index 1ec1410..257cd3e 100644
--- a/adapters/devices/sofar/device.go
+++ b/adapters/devices/sofar/device.go
@@ -28,8 +28,20 @@ func NewSofarLogger(serialNumber uint, connPort ports.CommunicationPort, attrWhi
 	}
 }
 
+func (s *Logger) nameFilter(k string) bool {
+	ok := len(s.attrWhiteList) == 0
+	if !ok { // TODO: also handle attrBlackList
+		_, ok = s.attrWhiteList[k]
+	}
+	return ok
+}
+
+func (s *Logger) GetDiscoveryFields() []ports.DiscoveryField {
+	return getDiscoveryFields(s.nameFilter)
+}
+
 func (s *Logger) Query() (map[string]interface{}, error) {
-	return readData(s.connPort, s.serialNumber, s.attrWhiteList, s.attrBlackList)
+	return readData(s.connPort, s.serialNumber, s.nameFilter)
 }
 
 func (s *Logger) Name() string {
diff --git a/adapters/devices/sofar/lsw.go b/adapters/devices/sofar/lsw.go
index f1c1a9f..bf6ac95 100644
--- a/adapters/devices/sofar/lsw.go
+++ b/adapters/devices/sofar/lsw.go
@@ -73,7 +73,7 @@ func (l LSWRequest) checksum(buf []byte) uint8 {
 	return checksum
 }
 
-func readData(connPort ports.CommunicationPort, serialNumber uint, attrWhiteList map[string]struct{}, attrBlackList []string) (map[string]interface{}, error) {
+func readData(connPort ports.CommunicationPort, serialNumber uint, nameFilter func(string) bool) (map[string]interface{}, error) {
 	result := make(map[string]interface{})
 
 	for _, rr := range allRegisterRanges {
@@ -82,12 +82,7 @@ func readData(connPort ports.CommunicationPort, serialNumber uint, attrWhiteList
 			return nil, err
 		}
 		for k, v := range reply {
-			ok := len(attrWhiteList) == 0
-			if !ok { // TODO: also handle attrBlackList
-				_, ok = attrWhiteList[k]
-			}
-			// log.Printf("readData: %s %v", k, ok)
-			if ok {
+			if nameFilter(k) {
 				result[k] = v
 			}
 		}
diff --git a/adapters/devices/sofar/sofar_protocol.go b/adapters/devices/sofar/sofar_protocol.go
index 369fd34..0cfd653 100644
--- a/adapters/devices/sofar/sofar_protocol.go
+++ b/adapters/devices/sofar/sofar_protocol.go
@@ -1,5 +1,9 @@
 package sofar
 
+import (
+	"github.com/kubaceg/sofar_g3_lsw3_logger_reader/ports"
+)
+
 type field struct {
 	register  int
 	name      string
@@ -37,6 +41,18 @@ func GetAllRegisterNames() []string {
 	return result
 }
 
+func getDiscoveryFields(nameFilter func(string) bool) []ports.DiscoveryField {
+	result := make([]ports.DiscoveryField, 0)
+	for _, rr := range allRegisterRanges {
+		for _, f := range rr.replyFields {
+			if f.name != "" && f.valueType != "" && nameFilter(f.name) {
+				result = append(result, ports.DiscoveryField{Name: f.name, Factor: f.factor, Unit: f.unit})
+			}
+		}
+	}
+	return result
+}
+
 var rrSystemInfo = registerRange{
 	start: 0x400,
 	end:   0x43a,
diff --git a/adapters/export/mosquitto/mqtt.go b/adapters/export/mosquitto/mqtt.go
index 8a7a4fd..85fba50 100644
--- a/adapters/export/mosquitto/mqtt.go
+++ b/adapters/export/mosquitto/mqtt.go
@@ -4,21 +4,24 @@ import (
 	"encoding/json"
 	"fmt"
 	"log"
+	"strings"
 	"time"
 
 	mqtt "github.com/eclipse/paho.mqtt.golang"
+	"github.com/kubaceg/sofar_g3_lsw3_logger_reader/ports"
 )
 
 type MqttConfig struct {
-	Url      string `yaml:"url"`
-	User     string `yaml:"user"`
-	Password string `yaml:"password"`
-	Prefix   string `yaml:"prefix"`
+	Url       string `yaml:"url"`
+	User      string `yaml:"user"`
+	Password  string `yaml:"password"`
+	Discovery string `yaml:"discovery"`
+	State     string `yaml:"state"`
 }
 
 type Connection struct {
 	client mqtt.Client
-	prefix string
+	state  string
 }
 
 var connectHandler mqtt.OnConnectHandler = func(client mqtt.Client) {
@@ -46,7 +49,7 @@ func New(config *MqttConfig) (*Connection, error) {
 
 	conn := &Connection{}
 	conn.client = mqtt.NewClient(opts)
-	conn.prefix = config.Prefix
+	conn.state = config.State
 	if token := conn.client.Connect(); token.Wait() && token.Error() != nil {
 		return nil, token.Error()
 	}
@@ -55,41 +58,58 @@ func New(config *MqttConfig) (*Connection, error) {
 
 }
 
-func publish(conn *Connection, k string, v interface{}) {
-	token := conn.client.Publish(fmt.Sprintf("%s/%s", conn.prefix, k), 0, true, fmt.Sprintf("%v", v))
+func (conn *Connection) publish(topic string, msg string, retain bool) {
+	token := conn.client.Publish(topic, 0, retain, msg)
 	res := token.WaitTimeout(1 * time.Second)
 	if !res || token.Error() != nil {
 		log.Printf("error inserting to MQTT: %s", token.Error())
 	}
 }
 
-// next thing to do is add discovery
-// func discovery() {
-// MQTT Discovery: https://www.home-assistant.io/integrations/mqtt/#mqtt-discovery
-
-// homeassistant/sensor/inverter/config
-// payload
-// unique_id: PV_Generation_Today01ad
-// state_topic: "homeassistant/sensor/inverter/PV_Generation_Today"
-// device_class: energy
-// state_class: measurement
-// unit_of_measurement: 'kWh'
-// value_template: "{{ value|int * 0.01 }}"```
+// return "power" for kW etc., "energy" for kWh etc.
+func unit2DeviceClass(unit string) string {
+	if strings.HasSuffix(unit, "Wh") {
+		return "energy"
+	} else if strings.HasSuffix(unit, "W") {
+		return "power"
+	} else {
+		return ""
+	}
+}
 
-// {"name": null, "device_class": "motion", "state_topic": "homeassistant/binary_sensor/garden/state", "unique_id": "motion01ad", "device": {"identifiers": ["01ad"], "name": "Garden" }}
-//}
+// MQTT Discovery: https://www.home-assistant.io/integrations/mqtt/#mqtt-discovery
+func (conn *Connection) InsertDiscoveryRecord(discovery string, state string, fields []ports.DiscoveryField) error {
+	uniq := "01ad" // TODO: get from config?
+	for _, f := range fields {
+		topic := fmt.Sprintf("%s/%s/config", discovery, f.Name)
+		json, _ := json.Marshal(map[string]interface{}{
+			"name":         f.Name,
+			"unique_id":    fmt.Sprintf("%s_%s", f.Name, uniq),
+			"device_class": unit2DeviceClass(f.Unit),
+			// "state_class": "measurement",
+			"state_topic":         state,
+			"unit_of_measurement": f.Unit,
+			"value_template":      fmt.Sprintf("{{ value_json.%s|int * %s }}", f.Name, f.Factor),
+			"device": map[string]interface{}{
+				"identifiers": [...]string{fmt.Sprintf("Inverter_%s", uniq)},
+				"name":        "Inverter",
+			},
+		})
+		conn.publish(topic, string(json), true) // MQTT Discovery messages should be retained, but in dev it can become a pain
+	}
+	return nil
+}
 
 func (conn *Connection) InsertRecord(measurement map[string]interface{}) error {
+	// make a copy
 	m := make(map[string]interface{}, len(measurement))
 	for k, v := range measurement {
 		m[k] = v
 	}
+	// add LastTimestamp
 	m["LastTimestamp"] = time.Now().UnixNano() / int64(time.Millisecond)
-	all, _ := json.Marshal(m)
-	m["All"] = string(all)
-	for k, v := range m {
-		publish(conn, k, v)
-	}
+	json, _ := json.Marshal(m)
+	conn.publish(conn.state, string(json), false) // state messages should not be retained
 	return nil
 }
 
diff --git a/config-example.yaml b/config-example.yaml
index 064b326..5945295 100644
--- a/config-example.yaml
+++ b/config-example.yaml
@@ -7,11 +7,12 @@ inverter:
   #
   # if there is a non-empty attrWhiteList then only these explicitly listed attributes are output
   attrWhiteList:
-    - ActivePower_Output_Total # Total PV generation (10W)
+    - ActivePower_Output_Total # Total PV generation (in units of 10W)
     - Power_PV1                # PV output of string 1 (10W)
     - Power_PV2                # PV output of string 2 (10W); PV1 + PV2 > ActivePower_Output_Total by about 3.5% due to inverter inefficiency?
     - ActivePower_Load_Sys     # total power consumption (10W)
     - ActivePower_PCC_Total    # grid export (+) or import (-) (10W) = ActivePower_Output_Total - ActivePower_Load_Sys (so it is redundant data)
+    - PV_Generation_Today      # generation since midnight (10Wh)
   # else attributes are output unless they match a regex in attrBlackList
   attrBlackList:
     - "^[ST]_"                 # prefix R_, S_, T_ for 3 phases, only R_ used in single phase systems
@@ -21,7 +22,8 @@ mqtt:                     # MQTT disabled if url & prefix both blank
   url: 1.2.3.4:1883       # MQTT broker URL (e.g. 1.2.3.4:1883)
   user:                   # MQTT username (leave empty when not needed)
   password:               # MQTT password (leave empty when not needed)
-  prefix: /sensors/energy/inverter # topic prefix on which data will be sent
+  discovery: homeassistant/sensor   # topic prefix for MQTT Discovery
+  state:     energy/inverter/state  # topic for state
 
 otlp:                     # OTLP disabled if both urls blank
   grpc:
diff --git a/main.go b/main.go
index 0d8c9f0..15d6088 100644
--- a/main.go
+++ b/main.go
@@ -14,7 +14,6 @@ import (
 	"github.com/kubaceg/sofar_g3_lsw3_logger_reader/adapters/devices/sofar"
 	"github.com/kubaceg/sofar_g3_lsw3_logger_reader/adapters/export/mosquitto"
 	"github.com/kubaceg/sofar_g3_lsw3_logger_reader/adapters/export/otlp"
-
 	"github.com/kubaceg/sofar_g3_lsw3_logger_reader/ports"
 )
 
@@ -40,7 +39,7 @@ func initialize() {
 		log.Fatalln(err)
 	}
 
-	hasMQTT = config.Mqtt.Url != "" && config.Mqtt.Prefix != ""
+	hasMQTT = config.Mqtt.Url != "" && config.Mqtt.State != ""
 	hasOTLP = config.Otlp.Grpc.Url != "" || config.Otlp.Http.Url != ""
 
 	if isSerialPort(config.Inverter.Port) {
@@ -71,6 +70,11 @@ func initialize() {
 
 func main() {
 	initialize()
+
+	if hasMQTT {
+		mqtt.InsertDiscoveryRecord(config.Mqtt.Discovery, config.Mqtt.State, device.GetDiscoveryFields())
+	}
+
 	failedConnections := 0
 
 	for {
@@ -92,35 +96,26 @@ func main() {
 		failedConnections = 0
 
 		if hasMQTT {
-			go func() {
-				err = mqtt.InsertRecord(measurements)
-				if err != nil {
-					log.Printf("failed to insert record to MQTT: %s\n", err)
-				} else {
-					log.Println("measurements pushed to MQTT")
-				}
-			}()
+			// removed from async go func 'goroutine', not needed and proper usage requires WaitGroup to wait for completion
+			mqtt.InsertRecord(measurements) // logs errors, always returns nil
 		}
 
 		if hasOTLP {
-			go func() {
-				err = telem.CollectAndPushMetrics(context.Background(), measurements)
-				if err != nil {
-					log.Printf("error recording telemetry: %s\n", err)
-				} else {
-					log.Println("measurements pushed via OLTP")
-				}
-			}()
+			// removed from async go func 'goroutine'
+			err = telem.CollectAndPushMetrics(context.Background(), measurements)
+			if err != nil {
+				log.Printf("error recording telemetry: %s\n", err)
+			} else {
+				log.Println("measurements pushed via OLTP")
+			}
 		}
 
+		// if mqtt & otlp were done async then the WaitGroup to wait for completion would go here
 		duration := time.Since(timeStart)
-
 		delay := time.Duration(config.Inverter.ReadInterval)*time.Second - duration
-		if delay <= 0 {
-			delay = 1 * time.Second
+		if delay > 0 {
+			time.Sleep(delay)
 		}
-
-		time.Sleep(delay)
 	}
 
 }
diff --git a/ports/database.go b/ports/database.go
index 8f5557c..0969b42 100644
--- a/ports/database.go
+++ b/ports/database.go
@@ -3,6 +3,7 @@ package ports
 import mqtt "github.com/eclipse/paho.mqtt.golang"
 
 type Database interface {
+	InsertDiscoveryRecord(discovery string, prefix string, fields []DiscoveryField) error
 	InsertRecord(measurement map[string]interface{}) error
 }
 
diff --git a/ports/devices.go b/ports/devices.go
index 6a8821d..566b9b7 100644
--- a/ports/devices.go
+++ b/ports/devices.go
@@ -1,6 +1,14 @@
 package ports
 
+// support MQTT Discovery
+type DiscoveryField struct {
+	Name   string
+	Factor string
+	Unit   string
+}
+
 type Device interface {
 	Name() string
+	GetDiscoveryFields() []DiscoveryField
 	Query() (map[string]interface{}, error)
 }
diff --git a/sh/cleanmqtt.sh b/sh/cleanmqtt.sh
new file mode 100644
index 0000000..702a8d5
--- /dev/null
+++ b/sh/cleanmqtt.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+# delete all "retained" messages
+echo "cleaning " $1 " :: usage: cleanmqtt <host>"
+host="$1"
+# get your MQTT credentials from HA UI: Settings > Devices & Services > MQTT > Configure > Reconfigure MQTT
+user=homeassistant
+paswd="yourPassword"
+
+mosquitto_sub -h "$host" -u "$user" -P "$paswd" -t "#" -v --retained-only | while read topic value
+do
+  echo "cleaning topic $topic"
+  mosquitto_pub -h "$host" -u "$user" -P "$paswd" -t "$topic" -r -n
+done

From 735a892c851f77b551d9d8d80f259b70dce7b7f3 Mon Sep 17 00:00:00 2001
From: Neil Bacon <2782001+neilbacon@users.noreply.github.com>
Date: Tue, 29 Aug 2023 18:04:20 +1000
Subject: [PATCH 03/16] custom component

---
 .gitignore                                    |  4 +--
 adapters/export/mosquitto/mqtt.go             | 25 +++++++--------
 .../sofar_g3_lsw3_logger_reader/README.md     | 32 +++++++++++++++++++
 .../sofar_g3_lsw3_logger_reader/__init__.py   | 20 ++++++++++++
 .../sofar_g3_lsw3_logger_reader/manifest.json |  9 ++++++
 5 files changed, 74 insertions(+), 16 deletions(-)
 create mode 100644 custom_components/sofar_g3_lsw3_logger_reader/README.md
 create mode 100644 custom_components/sofar_g3_lsw3_logger_reader/__init__.py
 create mode 100644 custom_components/sofar_g3_lsw3_logger_reader/manifest.json

diff --git a/.gitignore b/.gitignore
index 4c8c71c..5ae1223 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,6 +1,4 @@
-/sofar
-sofar_g3_lsw3_logger_reader
 config.yaml
 sofar
 sofar-arm
-.idea
\ No newline at end of file
+.idea
diff --git a/adapters/export/mosquitto/mqtt.go b/adapters/export/mosquitto/mqtt.go
index 85fba50..a775990 100644
--- a/adapters/export/mosquitto/mqtt.go
+++ b/adapters/export/mosquitto/mqtt.go
@@ -4,7 +4,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"log"
-	"strings"
 	"time"
 
 	mqtt "github.com/eclipse/paho.mqtt.golang"
@@ -67,15 +66,15 @@ func (conn *Connection) publish(topic string, msg string, retain bool) {
 }
 
 // return "power" for kW etc., "energy" for kWh etc.
-func unit2DeviceClass(unit string) string {
-	if strings.HasSuffix(unit, "Wh") {
-		return "energy"
-	} else if strings.HasSuffix(unit, "W") {
-		return "power"
-	} else {
-		return ""
-	}
-}
+// func unit2DeviceClass(unit string) string {
+// 	if strings.HasSuffix(unit, "Wh") {
+// 		return "energy"
+// 	} else if strings.HasSuffix(unit, "W") {
+// 		return "power"
+// 	} else {
+// 		return ""
+// 	}
+// }
 
 // MQTT Discovery: https://www.home-assistant.io/integrations/mqtt/#mqtt-discovery
 func (conn *Connection) InsertDiscoveryRecord(discovery string, state string, fields []ports.DiscoveryField) error {
@@ -83,9 +82,9 @@ func (conn *Connection) InsertDiscoveryRecord(discovery string, state string, fi
 	for _, f := range fields {
 		topic := fmt.Sprintf("%s/%s/config", discovery, f.Name)
 		json, _ := json.Marshal(map[string]interface{}{
-			"name":         f.Name,
-			"unique_id":    fmt.Sprintf("%s_%s", f.Name, uniq),
-			"device_class": unit2DeviceClass(f.Unit),
+			"name":      f.Name,
+			"unique_id": fmt.Sprintf("%s_%s", f.Name, uniq),
+			// "device_class": unit2DeviceClass(f.Unit),  // TODO: not working, always "energy"
 			// "state_class": "measurement",
 			"state_topic":         state,
 			"unit_of_measurement": f.Unit,
diff --git a/custom_components/sofar_g3_lsw3_logger_reader/README.md b/custom_components/sofar_g3_lsw3_logger_reader/README.md
new file mode 100644
index 0000000..36c8e2e
--- /dev/null
+++ b/custom_components/sofar_g3_lsw3_logger_reader/README.md
@@ -0,0 +1,32 @@
+# Custom Component to run sofar_g3_lsw3_logger_reader
+
+## Introduction
+
+`sofar_g3_lsw3_logger_reader` is a program written in go that:
+ - loops forever, polling the LSW3 data logger and writing the data to a MQTT topic
+ - reads its `config.yaml` from the current working directory
+ - writes logging to stderr
+ - supports MQTT Discovery
+
+ This component runs `sofar_g3_lsw3_logger_reader` with:
+  - the current working directory set to /config/custom_components/sofar_g3_lsw3_logger_reader
+  - stdout/err going to files out/err.log (because I haven't succeeded in integrating it with homeassistant logging)
+
+## Installation
+
+1. First get `sofar_g3_lsw3_logger_reader` working on your dev machine, so that you have a working `config.yaml` and `sofar` (x86) and `sofar-arm` (arm) binary executables.
+2. Copy this directory `custom_components/sofar_g3_lsw3_logger_reader` to homeassistant's `/config/custom_components` 
+(creating `/config/custom_components/sofar_g3_lsw3_logger_reader`). 
+You can use the `Samba share` or `Advanced SSH & Web Terminal` add-ons to do this.
+3. Copy the working `config.yaml` (from step 1) to the same directory.
+4. Copy the `sofar` (x86) or `sofar-arm` (arm) binary executable (from step 1) to `sofar` in this same directory.
+5. Enable the custom component by adding a line `sofar_g3_lsw3_logger_reader:` to homeassistant's `/config/configuration.yaml`.
+6. Do a full restart of homeassistant: `Developer Tools` > `YAML` > `CHECK CONFIGURATION` then `RESTART` > `Restart Home Assistant`
+7. Check the content of `err.log` in this same directory.
+8. Add the `Inverter` device to your dashhboard: `Settings` > `Devices & Services` > `Integration` > `MQTT` > `1 device` > `ADD TO DASHBOARD`
+
+## To Do
+
+1. Add to HACS
+2. Get logging going to homeassistant's log.
+3. Add support for "config flow" 
diff --git a/custom_components/sofar_g3_lsw3_logger_reader/__init__.py b/custom_components/sofar_g3_lsw3_logger_reader/__init__.py
new file mode 100644
index 0000000..d414774
--- /dev/null
+++ b/custom_components/sofar_g3_lsw3_logger_reader/__init__.py
@@ -0,0 +1,20 @@
+"""Integration for Sofar Solar PV inverters with LSW-3 WiFi sticks with SN 23XXXXXXX"""
+
+import subprocess
+import logging
+import os
+
+DOMAIN = "sofar_g3_lsw3_logger_reader"
+_LOGGER = logging.getLogger(__name__)
+
+def setup(hass, config):
+    hass.states.set("sofar_g3_lsw3_logger_reader.status", __name__)
+    _LOGGER.warning("setup: Working Dir is %s", os.getcwd())
+    _LOGGER.info("setup: Working Dir is %s", os.getcwd())  # main configuration.yaml should enable info, but not yet working  
+
+    out = open(f"/config/custom_components/{DOMAIN}/out.log", "w") # empty
+    err = open(f"/config/custom_components/{DOMAIN}/err.log", "w") # go logging package writes to stderr
+    proc = subprocess.Popen(f"/config/custom_components/{DOMAIN}/sofar", cwd=f"/config/custom_components/{DOMAIN}", stdout=out, stderr=err)
+    # the sofar process is left running forever
+    _LOGGER.warning("setup: end")
+    return True
diff --git a/custom_components/sofar_g3_lsw3_logger_reader/manifest.json b/custom_components/sofar_g3_lsw3_logger_reader/manifest.json
new file mode 100644
index 0000000..ee83873
--- /dev/null
+++ b/custom_components/sofar_g3_lsw3_logger_reader/manifest.json
@@ -0,0 +1,9 @@
+{
+  "domain": "sofar_g3_lsw3_logger_reader",
+  "name": "Sofar LSW3",
+  "version": "0.1.0",
+  "integration_type": "device",
+  "iot_class": "local_polling",
+  "dependencies": ["mqtt"],
+  "documentation": "https://github.com/neilbacon/sofar_g3_lsw3_logger_reader"
+}

From 711d09ca39da53c1cd6bc6864ebfcf043db7fdeb Mon Sep 17 00:00:00 2001
From: Neil Bacon <2782001+neilbacon@users.noreply.github.com>
Date: Sun, 17 Sep 2023 15:55:55 +1000
Subject: [PATCH 04/16] hacs

---
 .gitignore                                    |  2 -
 Makefile                                      |  8 ++--
 README.md                                     | 43 +++++++++++++------
 adapters/devices/sofar/device.go              | 38 +++++++++++++---
 adapters/export/mosquitto/mqtt.go             | 25 +++++------
 config.go                                     |  1 +
 .../sofar_g3_lsw3_logger_reader/README.md     | 19 +++-----
 .../sofar_g3_lsw3_logger_reader/__init__.py   | 14 +++---
 .../config-example.yaml                       |  7 +--
 .../sofar_g3_lsw3_logger_reader/manifest.json |  2 +-
 hacs.json                                     |  4 ++
 info.md                                       | 29 +++++++++++++
 main.go                                       |  4 +-
 13 files changed, 136 insertions(+), 60 deletions(-)
 rename config-example.yaml => custom_components/sofar_g3_lsw3_logger_reader/config-example.yaml (84%)
 create mode 100644 hacs.json
 create mode 100644 info.md

diff --git a/.gitignore b/.gitignore
index 5ae1223..60c9253 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,4 +1,2 @@
 config.yaml
-sofar
-sofar-arm
 .idea
diff --git a/Makefile b/Makefile
index c3bf0d3..c75cc2f 100644
--- a/Makefile
+++ b/Makefile
@@ -1,5 +1,7 @@
+all: build-arm build-x86
+
 build-arm:
-	env GOOS=linux GOARCH=arm GOARM=5 go build -o sofar-arm
+	env GOOS=linux GOARCH=arm GOARM=5 go build -o custom_components/sofar_g3_lsw3_logger_reader/sofar-arm
 
-build:
-	go build -o sofar
\ No newline at end of file
+build-x86:
+	go build -o custom_components/sofar_g3_lsw3_logger_reader/sofar-x86
\ No newline at end of file
diff --git a/README.md b/README.md
index b2d1203..542e6a4 100644
--- a/README.md
+++ b/README.md
@@ -1,24 +1,41 @@
 # Sofar g3 LSW-3 logger reader
-Tool written in GO for reading metrics from Sofar LSW-3 and writing results into MQTT topics. 
-Program queries logger modbus port in infinite loop and sends data into MQTT topics (e.g. mosquito in HomeAssistant).
+
+`sofar_g3_lsw3_logger_reader` is a go program to integrate certain Sofar solar PV inverters with MQTT (e.g. mosquito in HomeAssistant).
+It works with Sofar inverters fitted with a LSW-3 WiFi data logger with serial number 23XXXXXXXX.
+The program:
+ - loops forever, polling the modbus port on the LSW-3 and writing the data to MQTT (all local, no dependency on SolarMan cloud),
+ - reads its `config.yaml` from the current working directory,
+ - writes logging to stderr,
+ - supports MQTT Discovery.
+
+`custom_components/sofar_g3_lsw3_logger_reader` is a HomeAssistant custom component to run the above program.
 
 ## Installation and setup
-1. Download go 1.19
-2. Clone this repo `git clone git@github.com:kubaceg/sofar_g3_lsw3_logger_reader.git`
+
+1. Download go 1.19 - 1.21
+2. Clone this repo `git clone git@github.com:neilbacon/sofar_g3_lsw3_logger_reader.git`
 3. Go into project directory `cd sofar_g3_lsw3_logger_reader`
-4. Copy example config `cp config-example.yaml config.yaml`
-5. Edit `config.yaml` in Your favorite editor, fill all required stuff
-6. Build program `make build` or build for ARM machines e.g. raspberryPi `make build-arm`
-7. Run `./sofar` or `sofar-arm`
+6. Build program `make` (creates binaries in `custom_components/sofar_g3_lsw3_logger_reader`)
+4. Go into directory `custom_components/sofar_g3_lsw3_logger_reader` and copy example config `cp config-example.yaml config.yaml` and edit `config.yaml` to suit your needs
+7. Run `./sofar-x86` on x86 or `./sofar-arm` on arm (including Raspberry Pi)
+
+To run it as part of Home Assistant see the [custom component README](custom_components/sofar_g3_lsw3_logger_reader/README.md).
 
 ## Output data format
+
 ### MQTT
-Data will be sent into MQTT topic with name `{mqttPrefix}/{fieldName}` where:
-* mqttPrefix is value defined in `config.yaml` e.g. `/sensors/energy/inverter`
-* fieldName is measurement name, all available measurements are described in `adapters/devices/sofar/sofar_protocol.go`, e.g. `PV_Generation_Today`
 
-Full topic name for given example values is `/sensors/energy/inverter/PV_Generation_Today`.
-Additional field is `All` which contains all measurements and their values marshalled into one json.
+#### Attribute Filtering
+The LSW-3 provides a large number of attributes that you are most likely not interested in, so `config.yaml` allows you to filter them using either a white list or a black list used only if the white list is empty. The white list contains the attribute names (in full) to include and the black list contains regular expressions for attributes to exclude.
+
+#### MQTT Discovery
+On startup a message is sent (with `retain=true`) on a configuration topic for each data attribute. Home Assistant uses this information to configure an entity to manage the data attribute, removing the need for much manual configuration. The topic used is `{mqtt.discovery}/{attribute}/config` where `{mqtt.discovery}` comes from `config.yaml` and `{attribute}` is the attribute name. The JSON payload is described in the MQTT Discovery documentation.
+
+`retain=true` causes a newly connected client (such as a restarting Home Assistant) to receive a copy of these messages. MQTT Discovery documentation suggests this is a good idea, but in development it can be a pain, leaving old messages hanging around indefinitely. You can use the `sh/cleanmqtt.sh` utility to clean up these messages. It depends on a MQTT installation and can be run in the HA OS's mosquitto container `addon_core_mosquitto`.
+
+#### MQTT Data
+
+All attributes are sent in a single message (with retain=false) with JSON payload to the topic `{mqtt.state}` specified in `config.yaml`.
 
 ### OTLP
 Data can also be sent over OTLP protocol to a gRPC or http server. Typically, this would be received by the 
diff --git a/adapters/devices/sofar/device.go b/adapters/devices/sofar/device.go
index 257cd3e..e3b8977 100644
--- a/adapters/devices/sofar/device.go
+++ b/adapters/devices/sofar/device.go
@@ -1,12 +1,17 @@
 package sofar
 
-import "github.com/kubaceg/sofar_g3_lsw3_logger_reader/ports"
+import (
+	"log"
+	"regexp"
+
+	"github.com/kubaceg/sofar_g3_lsw3_logger_reader/ports"
+)
 
 type Logger struct {
 	serialNumber  uint
 	connPort      ports.CommunicationPort
 	attrWhiteList map[string]struct{}
-	attrBlackList []string
+	attrBlackList []*regexp.Regexp
 }
 
 // for a set in go we use a map of keys -> empty struct
@@ -19,21 +24,40 @@ func toSet(slice []string) map[string]struct{} {
 	return set
 }
 
+func toREs(patterns []string) []*regexp.Regexp {
+	res := make([]*regexp.Regexp, len(patterns))
+	for idx, p := range patterns {
+		re, err := regexp.Compile(p)
+		if err == nil {
+			res = append(res, re)
+		} else {
+			log.Printf("config attrBlackList item %d '%s' not a valid regexp; %v", idx, p, err)
+		}
+	}
+	return res
+}
+
 func NewSofarLogger(serialNumber uint, connPort ports.CommunicationPort, attrWhiteList []string, attrBlackList []string) *Logger {
 	return &Logger{
 		serialNumber:  serialNumber,
 		connPort:      connPort,
 		attrWhiteList: toSet(attrWhiteList),
-		attrBlackList: attrBlackList,
+		attrBlackList: toREs(attrBlackList),
 	}
 }
 
 func (s *Logger) nameFilter(k string) bool {
-	ok := len(s.attrWhiteList) == 0
-	if !ok { // TODO: also handle attrBlackList
-		_, ok = s.attrWhiteList[k]
+	if len(s.attrWhiteList) > 0 {
+		_, ok := s.attrWhiteList[k]
+		return ok
+	} else {
+		for _, re := range s.attrBlackList {
+			if re.MatchString(k) {
+				return false
+			}
+		}
 	}
-	return ok
+	return true
 }
 
 func (s *Logger) GetDiscoveryFields() []ports.DiscoveryField {
diff --git a/adapters/export/mosquitto/mqtt.go b/adapters/export/mosquitto/mqtt.go
index a775990..85fba50 100644
--- a/adapters/export/mosquitto/mqtt.go
+++ b/adapters/export/mosquitto/mqtt.go
@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"log"
+	"strings"
 	"time"
 
 	mqtt "github.com/eclipse/paho.mqtt.golang"
@@ -66,15 +67,15 @@ func (conn *Connection) publish(topic string, msg string, retain bool) {
 }
 
 // return "power" for kW etc., "energy" for kWh etc.
-// func unit2DeviceClass(unit string) string {
-// 	if strings.HasSuffix(unit, "Wh") {
-// 		return "energy"
-// 	} else if strings.HasSuffix(unit, "W") {
-// 		return "power"
-// 	} else {
-// 		return ""
-// 	}
-// }
+func unit2DeviceClass(unit string) string {
+	if strings.HasSuffix(unit, "Wh") {
+		return "energy"
+	} else if strings.HasSuffix(unit, "W") {
+		return "power"
+	} else {
+		return ""
+	}
+}
 
 // MQTT Discovery: https://www.home-assistant.io/integrations/mqtt/#mqtt-discovery
 func (conn *Connection) InsertDiscoveryRecord(discovery string, state string, fields []ports.DiscoveryField) error {
@@ -82,9 +83,9 @@ func (conn *Connection) InsertDiscoveryRecord(discovery string, state string, fi
 	for _, f := range fields {
 		topic := fmt.Sprintf("%s/%s/config", discovery, f.Name)
 		json, _ := json.Marshal(map[string]interface{}{
-			"name":      f.Name,
-			"unique_id": fmt.Sprintf("%s_%s", f.Name, uniq),
-			// "device_class": unit2DeviceClass(f.Unit),  // TODO: not working, always "energy"
+			"name":         f.Name,
+			"unique_id":    fmt.Sprintf("%s_%s", f.Name, uniq),
+			"device_class": unit2DeviceClass(f.Unit),
 			// "state_class": "measurement",
 			"state_topic":         state,
 			"unit_of_measurement": f.Unit,
diff --git a/config.go b/config.go
index be6ad9e..225f15a 100644
--- a/config.go
+++ b/config.go
@@ -15,6 +15,7 @@ type Config struct {
 		Port          string   `yaml:"port"`
 		LoggerSerial  uint     `yaml:"loggerSerial"`
 		ReadInterval  int      `default:"60" yaml:"readInterval"`
+		LoopLogging   bool     `default:"true" yaml:"loopLogging"`
 		AttrWhiteList []string `yaml:"attrWhiteList"`
 		AttrBlackList []string `yaml:"attrBlackList"`
 	} `yaml:"inverter"`
diff --git a/custom_components/sofar_g3_lsw3_logger_reader/README.md b/custom_components/sofar_g3_lsw3_logger_reader/README.md
index 36c8e2e..80e907c 100644
--- a/custom_components/sofar_g3_lsw3_logger_reader/README.md
+++ b/custom_components/sofar_g3_lsw3_logger_reader/README.md
@@ -1,25 +1,18 @@
-# Custom Component to run sofar_g3_lsw3_logger_reader
+# Sofar g3 LSW-3 logger reader Custom Component
 
-## Introduction
-
-`sofar_g3_lsw3_logger_reader` is a program written in go that:
- - loops forever, polling the LSW3 data logger and writing the data to a MQTT topic
- - reads its `config.yaml` from the current working directory
- - writes logging to stderr
- - supports MQTT Discovery
-
- This component runs `sofar_g3_lsw3_logger_reader` with:
+This is a HomeAssistant custom component to run the [sofar_g3_lsw3_logger_reader](../../README.md) program.
+It runs the program with:
   - the current working directory set to /config/custom_components/sofar_g3_lsw3_logger_reader
   - stdout/err going to files out/err.log (because I haven't succeeded in integrating it with homeassistant logging)
 
-## Installation
+## Installation and setup
 
-1. First get `sofar_g3_lsw3_logger_reader` working on your dev machine, so that you have a working `config.yaml` and `sofar` (x86) and `sofar-arm` (arm) binary executables.
+1. First get the go program `sofar_g3_lsw3_logger_reader` working on your dev machine, so that you have a working `config.yaml` and `sofar-x86` (x86) and `sofar-arm` (arm) binary executables.
 2. Copy this directory `custom_components/sofar_g3_lsw3_logger_reader` to homeassistant's `/config/custom_components` 
 (creating `/config/custom_components/sofar_g3_lsw3_logger_reader`). 
 You can use the `Samba share` or `Advanced SSH & Web Terminal` add-ons to do this.
 3. Copy the working `config.yaml` (from step 1) to the same directory.
-4. Copy the `sofar` (x86) or `sofar-arm` (arm) binary executable (from step 1) to `sofar` in this same directory.
+4. Copy the executables `sofar` (x86) and `sofar-arm` (arm) (from step 1) to this same directory.
 5. Enable the custom component by adding a line `sofar_g3_lsw3_logger_reader:` to homeassistant's `/config/configuration.yaml`.
 6. Do a full restart of homeassistant: `Developer Tools` > `YAML` > `CHECK CONFIGURATION` then `RESTART` > `Restart Home Assistant`
 7. Check the content of `err.log` in this same directory.
diff --git a/custom_components/sofar_g3_lsw3_logger_reader/__init__.py b/custom_components/sofar_g3_lsw3_logger_reader/__init__.py
index d414774..9f2f0f2 100644
--- a/custom_components/sofar_g3_lsw3_logger_reader/__init__.py
+++ b/custom_components/sofar_g3_lsw3_logger_reader/__init__.py
@@ -1,5 +1,6 @@
 """Integration for Sofar Solar PV inverters with LSW-3 WiFi sticks with SN 23XXXXXXX"""
 
+import platform
 import subprocess
 import logging
 import os
@@ -8,13 +9,16 @@
 _LOGGER = logging.getLogger(__name__)
 
 def setup(hass, config):
-    hass.states.set("sofar_g3_lsw3_logger_reader.status", __name__)
-    _LOGGER.warning("setup: Working Dir is %s", os.getcwd())
-    _LOGGER.info("setup: Working Dir is %s", os.getcwd())  # main configuration.yaml should enable info, but not yet working  
+    hass.states.set("sofar_g3_lsw3_logger_reader.status", __name__) # custom_components.sofar_g3_lsw3_logger_reader
+    _LOGGER.warning("setup: Working Dir is %s", os.getcwd())        # /config
+    _LOGGER.info("setup: Working Dir is %s", os.getcwd())           # main configuration.yaml should enable info, but this is not yet working  
 
+    # When the (go lang) sofar process writes a line to stderr, I'd like it to go to _LOGGER.info(), but I haven't fugured out how to do this
+    # without blocking, so for now we connect stdout/err to log files.
     out = open(f"/config/custom_components/{DOMAIN}/out.log", "w") # empty
     err = open(f"/config/custom_components/{DOMAIN}/err.log", "w") # go logging package writes to stderr
-    proc = subprocess.Popen(f"/config/custom_components/{DOMAIN}/sofar", cwd=f"/config/custom_components/{DOMAIN}", stdout=out, stderr=err)
-    # the sofar process is left running forever
+    exe = f"/config/custom_components/{DOMAIN}/sofar-arm" if platform.processor.startsWith("arm") else f"/config/custom_components/{DOMAIN}/sofar-x86" 
+    proc = subprocess.Popen(exe, cwd=f"/config/custom_components/{DOMAIN}", stdout=out, stderr=err)
+    # the (go lang) sofar process reads its config.yaml from cwd, then loops forever
     _LOGGER.warning("setup: end")
     return True
diff --git a/config-example.yaml b/custom_components/sofar_g3_lsw3_logger_reader/config-example.yaml
similarity index 84%
rename from config-example.yaml
rename to custom_components/sofar_g3_lsw3_logger_reader/config-example.yaml
index 5945295..eae62d1 100644
--- a/config-example.yaml
+++ b/custom_components/sofar_g3_lsw3_logger_reader/config-example.yaml
@@ -1,9 +1,10 @@
 # make your own in config.yaml
 
 inverter:
-  port: 1.2.3.4:8899      # required, port name e.g. /dev/ttyUSB0 for serial or 1.2.3.4:8899 for TCP/IP
-  loggerSerial: 23XXXXXXX # required, logger serial number
-  readInterval: 60        # update interval in seconds, default 60
+  port: 1.2.3.4:8899       # required, port name e.g. /dev/ttyUSB0 for serial or 1.2.3.4:8899 for TCP/IP
+  loggerSerial: 23XXXXXXXX # required, logger serial number
+  readInterval: 60         # update interval in seconds, default 60
+  loopLogging: true        # false to avoid ever growing log file
   #
   # if there is a non-empty attrWhiteList then only these explicitly listed attributes are output
   attrWhiteList:
diff --git a/custom_components/sofar_g3_lsw3_logger_reader/manifest.json b/custom_components/sofar_g3_lsw3_logger_reader/manifest.json
index ee83873..1033cf0 100644
--- a/custom_components/sofar_g3_lsw3_logger_reader/manifest.json
+++ b/custom_components/sofar_g3_lsw3_logger_reader/manifest.json
@@ -3,7 +3,7 @@
   "name": "Sofar LSW3",
   "version": "0.1.0",
   "integration_type": "device",
-  "iot_class": "local_polling",
+  "iot_class": "local_push",
   "dependencies": ["mqtt"],
   "documentation": "https://github.com/neilbacon/sofar_g3_lsw3_logger_reader"
 }
diff --git a/hacs.json b/hacs.json
new file mode 100644
index 0000000..9e72c0f
--- /dev/null
+++ b/hacs.json
@@ -0,0 +1,4 @@
+{
+  "name": "Sofar LSW3",
+  "homeassistant": "0.116.0"
+}
diff --git a/info.md b/info.md
new file mode 100644
index 0000000..f9b61ae
--- /dev/null
+++ b/info.md
@@ -0,0 +1,29 @@
+# Integration for Sofar Solar PV Inverters with LSW-3 WiFi data logger with SN 23XXXXXXX
+
+This integration regularly polls the LSW-3 and sends inverter data on a MQTT topic.
+It supports MQTT discovery.
+It is implemented by a binary executable `sofar-arm` (on arm) or `sofar-x86` (on x86), which is kicked off when the integration starts and runs forever. 
+
+## Manual Installation
+
+1. First get the go program `sofar_g3_lsw3_logger_reader` working on your dev machine, so that you have a working `config.yaml` and `sofar-x86` (x86) and `sofar-arm` (arm) binary executables.
+2. Copy this directory `custom_components/sofar_g3_lsw3_logger_reader` to homeassistant's `/config/custom_components` 
+(creating `/config/custom_components/sofar_g3_lsw3_logger_reader`). 
+You can use the `Samba share` or `Advanced SSH & Web Terminal` add-ons to do this.
+3. Copy the working `config.yaml` (from step 1) to the same directory.
+4. Copy the executables `sofar` (x86) and `sofar-arm` (arm) (from step 1) to this same directory.
+5. Enable the custom component by adding a line `sofar_g3_lsw3_logger_reader:` to homeassistant's `/config/configuration.yaml`.
+6. Do a full restart of homeassistant: `Developer Tools` > `YAML` > `CHECK CONFIGURATION` then `RESTART` > `Restart Home Assistant`
+7. Check the content of `err.log` in this same directory.
+8. Add the `Inverter` device to your dashhboard: `Settings` > `Devices & Services` > `Integration` > `MQTT` > `1 device` > `ADD TO DASHBOARD`
+
+## HACS Installation
+
+1. Install using HACS
+2. In directory `/config/custom_components/sofar_g3_lsw3_logger_reader` copy `config-example.yaml` to `config.yaml` and edit to match your requirements.
+3. Restart Home Assistant to run the integration.
+
+## To Do
+
+1. Get logging going to homeassistant's log.
+2. Add support for "config flow" 
diff --git a/main.go b/main.go
index 15d6088..d96346a 100644
--- a/main.go
+++ b/main.go
@@ -78,7 +78,9 @@ func main() {
 	failedConnections := 0
 
 	for {
-		log.Printf("performing measurements")
+		if config.Inverter.LoopLogging {
+			log.Printf("performing measurements")
+		}
 		timeStart := time.Now()
 
 		measurements, err := device.Query()

From 7ab7810690db36f65515f1cadf982803fc613622 Mon Sep 17 00:00:00 2001
From: Neil Bacon <2782001+neilbacon@users.noreply.github.com>
Date: Wed, 20 Sep 2023 16:24:31 +1000
Subject: [PATCH 05/16] handle additional device_classes and Unavailable when
 inverter offline at night

---
 adapters/devices/sofar/device.go              |  2 +-
 adapters/devices/sofar/sofar_protocol.go      | 20 +++---
 adapters/export/mosquitto/mqtt.go             | 35 +++++++++--
 .../sofar_g3_lsw3_logger_reader/__init__.py   | 16 ++---
 .../config-example.yaml                       | 63 ++++++++++++++++++-
 main.go                                       |  4 +-
 ports/database.go                             |  2 +-
 7 files changed, 114 insertions(+), 28 deletions(-)

diff --git a/adapters/devices/sofar/device.go b/adapters/devices/sofar/device.go
index e3b8977..89c3976 100644
--- a/adapters/devices/sofar/device.go
+++ b/adapters/devices/sofar/device.go
@@ -25,7 +25,7 @@ func toSet(slice []string) map[string]struct{} {
 }
 
 func toREs(patterns []string) []*regexp.Regexp {
-	res := make([]*regexp.Regexp, len(patterns))
+	res := make([]*regexp.Regexp, 0, len(patterns))
 	for idx, p := range patterns {
 		re, err := regexp.Compile(p)
 		if err == nil {
diff --git a/adapters/devices/sofar/sofar_protocol.go b/adapters/devices/sofar/sofar_protocol.go
index 0cfd653..0ccb751 100644
--- a/adapters/devices/sofar/sofar_protocol.go
+++ b/adapters/devices/sofar/sofar_protocol.go
@@ -91,18 +91,18 @@ var rrSystemInfo = registerRange{
 		{0x0423, "Temp_Rsvd1", "I16", "1", "℃"},
 		{0x0424, "Temp_Rsvd2", "I16", "1", "℃"},
 		{0x0425, "Temp_Rsvd3", "I16", "1", "℃"},
-		{0x0426, "GenerationTime_Today", "U16", "1", "Minute"},
-		{0x0427, "GenerationTime_Total", "U32", "1", "Minute"},
+		{0x0426, "GenerationTime_Today", "U16", "1", "min"}, // HA uses d, h, min, s not Minute
+		{0x0427, "GenerationTime_Total", "U32", "1", "min"},
 		{0x0428, "", "", "", ""},
-		{0x0429, "ServiceTime_Total", "U32", "1", "Minute"},
+		{0x0429, "ServiceTime_Total", "U32", "1", "min"},
 		{0x042A, "", "", "", ""},
 		{0x042B, "InsulationResistance", "U16", "1", "kΩ"},
 		{0x042C, "SysTime_Year", "U16", "", ""},
 		{0x042D, "SysTime_Month", "U16", "", ""},
-		{0x042E, "SysTime_Date", "U16", "", ""},
-		{0x042F, "SysTime_Hour", "U16", "", ""},
-		{0x0430, "SysTime_Minute", "U16", "", ""},
-		{0x0431, "SysTime_Second", "U16", "", ""},
+		{0x042E, "SysTime_Date", "U16", "1", "d"},
+		{0x042F, "SysTime_Hour", "U16", "1", "h"},
+		{0x0430, "SysTime_Minute", "U16", "1", "min"},
+		{0x0431, "SysTime_Second", "U16", "1", "s"},
 		{0x0432, "Fault19", "U16", "", ""},
 		{0x0433, "Fault20", "U16", "", ""},
 		{0x0434, "Fault21", "U16", "", ""},
@@ -120,11 +120,11 @@ var rrEnergyTodayTotals = registerRange{
 	replyFields: []field{
 		{0x684, "PV_Generation_Today", "U32", "0.01", "kWh"},
 		{0x686, "PV_Generation_Total", "U32", "0.1", "kWh"},
-		{0x688, "Load_Consumption_Today", "U32", "0.1", "kWh"},
+		{0x688, "Load_Consumption_Today", "U32", "0.01", "kWh"},
 		{0x68A, "Load_Consumption_Total", "U32", "0.1", "kWh"},
-		{0x68C, "Energy_Purchase_Today", "U32", "0.1", "kWh"},
+		{0x68C, "Energy_Purchase_Today", "U32", "0.01", "kWh"},
 		{0x68E, "Energy_Purchase_Total", "U32", "0.1", "kWh"},
-		{0x690, "Energy_Selling_Today", "U32", "0.1", "kWh"},
+		{0x690, "Energy_Selling_Today", "U32", "0.01", "kWh"},
 		{0x692, "Energy_Selling_Total", "U32", "0.1", "kWh"},
 		{0x694, "Bat_Charge_Today", "U32", "0.01", "kWh"},
 		{0x696, "Bat_Charge_Total", "U32", "0.1", "kWh"},
diff --git a/adapters/export/mosquitto/mqtt.go b/adapters/export/mosquitto/mqtt.go
index 85fba50..341ddb9 100644
--- a/adapters/export/mosquitto/mqtt.go
+++ b/adapters/export/mosquitto/mqtt.go
@@ -72,24 +72,49 @@ func unit2DeviceClass(unit string) string {
 		return "energy"
 	} else if strings.HasSuffix(unit, "W") {
 		return "power"
+	} else if strings.HasSuffix(unit, "Hz") {
+		return "frequency"
+	} else if strings.HasSuffix(unit, "VA") {
+		return "apparent_power"
+	} else if strings.HasSuffix(unit, "VAR") {
+		return "reactive_power"
+	} else if strings.HasSuffix(unit, "V") {
+		return "voltage"
+	} else if strings.HasSuffix(unit, "A") {
+		return "current"
+	} else if strings.HasSuffix(unit, "Ω") {
+		return "voltage" // resistance not valid in https://developers.home-assistant.io/docs/core/entity/sensor/ so use "voltage"
+	} else if strings.HasSuffix(unit, "℃") {
+		return "temperature"
+	} else if strings.HasSuffix(unit, "min") {
+		return "duration"
 	} else {
 		return ""
 	}
 }
 
+func unit2StateClass(unit string) string {
+	if strings.HasSuffix(unit, "Wh") {
+		return "total"
+	} else {
+		return "measurement"
+	}
+}
+
 // MQTT Discovery: https://www.home-assistant.io/integrations/mqtt/#mqtt-discovery
-func (conn *Connection) InsertDiscoveryRecord(discovery string, state string, fields []ports.DiscoveryField) error {
+func (conn *Connection) InsertDiscoveryRecord(discovery string, state string, expireAfter int, fields []ports.DiscoveryField) error {
 	uniq := "01ad" // TODO: get from config?
 	for _, f := range fields {
 		topic := fmt.Sprintf("%s/%s/config", discovery, f.Name)
 		json, _ := json.Marshal(map[string]interface{}{
-			"name":         f.Name,
-			"unique_id":    fmt.Sprintf("%s_%s", f.Name, uniq),
-			"device_class": unit2DeviceClass(f.Unit),
-			// "state_class": "measurement",
+			"name":                f.Name,
+			"unique_id":           fmt.Sprintf("%s_%s", f.Name, uniq),
+			"device_class":        unit2DeviceClass(f.Unit),
+			"state_class":         unit2StateClass(f.Unit),
 			"state_topic":         state,
 			"unit_of_measurement": f.Unit,
 			"value_template":      fmt.Sprintf("{{ value_json.%s|int * %s }}", f.Name, f.Factor),
+			"expire_after":        expireAfter, // no messages for this long makes entity "Unavailable"
 			"device": map[string]interface{}{
 				"identifiers": [...]string{fmt.Sprintf("Inverter_%s", uniq)},
 				"name":        "Inverter",
diff --git a/custom_components/sofar_g3_lsw3_logger_reader/__init__.py b/custom_components/sofar_g3_lsw3_logger_reader/__init__.py
index 9f2f0f2..c5d38ec 100644
--- a/custom_components/sofar_g3_lsw3_logger_reader/__init__.py
+++ b/custom_components/sofar_g3_lsw3_logger_reader/__init__.py
@@ -5,20 +5,22 @@
 import logging
 import os
 
-DOMAIN = "sofar_g3_lsw3_logger_reader"
 _LOGGER = logging.getLogger(__name__)
+_DOMAIN = "sofar_g3_lsw3_logger_reader"
+_DIR = f"/config/custom_components/{_DOMAIN}"
 
 def setup(hass, config):
-    hass.states.set("sofar_g3_lsw3_logger_reader.status", __name__) # custom_components.sofar_g3_lsw3_logger_reader
+    hass.states.set(f"{_DOMAIN}.status", __name__)                  # custom_components.sofar_g3_lsw3_logger_reader
     _LOGGER.warning("setup: Working Dir is %s", os.getcwd())        # /config
     _LOGGER.info("setup: Working Dir is %s", os.getcwd())           # main configuration.yaml should enable info, but this is not yet working  
 
-    # When the (go lang) sofar process writes a line to stderr, I'd like it to go to _LOGGER.info(), but I haven't fugured out how to do this
+    # When the process writes a line to stderr, I'd like it to go to _LOGGER.info(), but I haven't fugured out how to do this
     # without blocking, so for now we connect stdout/err to log files.
-    out = open(f"/config/custom_components/{DOMAIN}/out.log", "w") # empty
-    err = open(f"/config/custom_components/{DOMAIN}/err.log", "w") # go logging package writes to stderr
-    exe = f"/config/custom_components/{DOMAIN}/sofar-arm" if platform.processor.startsWith("arm") else f"/config/custom_components/{DOMAIN}/sofar-x86" 
-    proc = subprocess.Popen(exe, cwd=f"/config/custom_components/{DOMAIN}", stdout=out, stderr=err)
+    out = open(f"{_DIR}/out.log", "w")                              # empty
+    err = open(f"{_DIR}/err.log", "w")                              # go logging package writes to stderr
+    # platform.processor() is '' on HA OS on Raspberry Pi
+    exe = f"{_DIR}/sofar-x86" if platform.processor().startswith("x86") else f"{_DIR}/sofar-arm" 
+    proc = subprocess.Popen(exe, cwd=f"{_DIR}", stdout=out, stderr=err)
     # the (go lang) sofar process reads its config.yaml from cwd, then loops forever
     _LOGGER.warning("setup: end")
     return True
diff --git a/custom_components/sofar_g3_lsw3_logger_reader/config-example.yaml b/custom_components/sofar_g3_lsw3_logger_reader/config-example.yaml
index eae62d1..1ba1308 100644
--- a/custom_components/sofar_g3_lsw3_logger_reader/config-example.yaml
+++ b/custom_components/sofar_g3_lsw3_logger_reader/config-example.yaml
@@ -4,16 +4,25 @@ inverter:
   port: 1.2.3.4:8899       # required, port name e.g. /dev/ttyUSB0 for serial or 1.2.3.4:8899 for TCP/IP
   loggerSerial: 23XXXXXXXX # required, logger serial number
   readInterval: 60         # update interval in seconds, default 60
-  loopLogging: true        # false to avoid ever growing log file
+  loopLogging: true        # false to avoid a line to log file every readInterval
   #
   # if there is a non-empty attrWhiteList then only these explicitly listed attributes are output
   attrWhiteList:
     - ActivePower_Output_Total # Total PV generation (in units of 10W)
-    - Power_PV1                # PV output of string 1 (10W)
-    - Power_PV2                # PV output of string 2 (10W); PV1 + PV2 > ActivePower_Output_Total by about 3.5% due to inverter inefficiency?
     - ActivePower_Load_Sys     # total power consumption (10W)
     - ActivePower_PCC_Total    # grid export (+) or import (-) (10W) = ActivePower_Output_Total - ActivePower_Load_Sys (so it is redundant data)
+    - Power_PV1                # PV output of string 1 (10W)
+    - Power_PV2                # PV output of string 2 (10W); PV1 + PV2 > ActivePower_Output_Total by about 3.5% due to inverter inefficiency?
     - PV_Generation_Today      # generation since midnight (10Wh)
+    - PV_Generation_Total
+    - GenerationTime_Today
+    - GenerationTime_Total
+    - Frequency_Grid
+    - Voltage_Phase_R          # grid voltage on phase R/1
+    - InsulationResistance     # changes to these last ones might indicate a problem
+    - Temperature_Env1
+    - Temperature_HeatSink1
+    - Temperature_HeatSink2
   # else attributes are output unless they match a regex in attrBlackList
   attrBlackList:
     - "^[ST]_"                 # prefix R_, S_, T_ for 3 phases, only R_ used in single phase systems
@@ -32,3 +41,51 @@ otlp:                     # OTLP disabled if both urls blank
   http:
     url:                  # URL for HTTP OLTP server e.g. 0.0.0.0:4318 
   prefix: sofar.logging
+
+# List of all attributes with non-zero values fetched from my inverter:
+#
+# # Most interesting to me:
+#     "ActivePower_Load_Sys": 16,        # total power consumption (10W)
+#     "ActivePower_Output_Total": 333,   # Total PV generation (in units of 10W)
+#     "ActivePower_PCC_Total": 317,      # grid export (+) or import (-) (10W) = ActivePower_Output_Total - ActivePower_Load_Sys (so it is redundant data)
+#     "GenerationTime_Today": 553,
+#     "GenerationTime_Total": 54992,
+#     "PV_Generation_Today": 3428,       # generation since midnight (10Wh)
+#     "PV_Generation_Total": 20355,
+
+# # Not credible:
+#     "Load_Consumption_Today": 3428,    # same as PV_Generation_Today, so not credible as Load
+#     "Load_Consumption_Total": 20355,   # same as PV_Generation_Total, so not credible as Load
+
+# # Inverter:
+#     "ApparentPower_Output_Total": 33,
+#     "Countdown": 60,
+#     "InsulationResistance": 1437,
+#     "ServiceTime_Total": 56198,
+#     "SysState": 2,
+#     "SysTime_Date": 19,
+#     "SysTime_Hour": 14,
+#     "SysTime_Minute": 56,
+#     "SysTime_Month": 9,
+#     "SysTime_Second": 1,
+#     "SysTime_Year": 23,
+#     "Temperature_Env1": 48,                # Single Plate Ambient Temperature (label from SolarMan app)
+#     "Temperature_HeatSink1": 35,           # Radiator Temperature 1 (label from SolarMan app)
+#     "Temperature_HeatSink2": 120,          # nothing with this value in SolarMan app
+
+# # PV strings of panels:
+#     "Power_PV1": 175,                      # PV output of string 1 (10W)
+#     "Power_PV2": 172,                      # PV output of string 2 (10W); PV1 + PV2 > ActivePower_Output_Total by about 3.5% due to inverter inefficiency?
+#     "Voltage_PV1": 2438,
+#     "Voltage_PV2": 2436,
+#     "Current_PV1": 720,
+#     "Current_PV2": 706,
+
+# # Grid:
+#     "Frequency_Grid": 5000,
+
+# # Phase R (single phase power):
+#     "ActivePower_PCC_R": 317,              # same as ActivePower_PCC_Total
+#     "Voltage_Phase_R": 2352                # grid voltage
+#     "Current_Output_R": 1418,
+#     "Current_PCC_R": 1394,
diff --git a/main.go b/main.go
index d96346a..db1c1ef 100644
--- a/main.go
+++ b/main.go
@@ -72,7 +72,7 @@ func main() {
 	initialize()
 
 	if hasMQTT {
-		mqtt.InsertDiscoveryRecord(config.Mqtt.Discovery, config.Mqtt.State, device.GetDiscoveryFields())
+		mqtt.InsertDiscoveryRecord(config.Mqtt.Discovery, config.Mqtt.State, config.Inverter.ReadInterval*5, device.GetDiscoveryFields())
 	}
 
 	failedConnections := 0
@@ -86,6 +86,8 @@ func main() {
 		measurements, err := device.Query()
 		if err != nil {
 			log.Printf("failed to perform measurements: %s", err)
+			// at night, inverter is offline, err = "dial tcp 192.168.xx.xxx:8899: i/o timeout"
+			// At other times we occaisionally get this error and also: "short reply: xx bytes"
 			failedConnections++
 
 			if failedConnections > maximumFailedConnections {
diff --git a/ports/database.go b/ports/database.go
index 0969b42..cf88106 100644
--- a/ports/database.go
+++ b/ports/database.go
@@ -3,7 +3,7 @@ package ports
 import mqtt "github.com/eclipse/paho.mqtt.golang"
 
 type Database interface {
-	InsertDiscoveryRecord(discovery string, prefix string, fields []DiscoveryField) error
+	InsertDiscoveryRecord(discovery string, prefix string, expireAfter int, fields []DiscoveryField) error
 	InsertRecord(measurement map[string]interface{}) error
 }
 

From 997f55449ac969e01a2e98c2cdf9a23ae54d51e5 Mon Sep 17 00:00:00 2001
From: Neil Bacon <2782001+neilbacon@users.noreply.github.com>
Date: Wed, 20 Sep 2023 16:25:46 +1000
Subject: [PATCH 06/16] sofar binaries

---
 .gitignore | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.gitignore b/.gitignore
index 60c9253..b909644 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,2 +1,4 @@
 config.yaml
+sofar-x86
+sofar-arm
 .idea

From 0788947024e2584ef688db056d8250ebf5f8f23f Mon Sep 17 00:00:00 2001
From: Neil Bacon <2782001+neilbacon@users.noreply.github.com>
Date: Wed, 20 Sep 2023 16:38:09 +1000
Subject: [PATCH 07/16] doc

---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 542e6a4..0ebedd7 100644
--- a/README.md
+++ b/README.md
@@ -1,9 +1,9 @@
 # Sofar g3 LSW-3 logger reader
 
-`sofar_g3_lsw3_logger_reader` is a go program to integrate certain Sofar solar PV inverters with MQTT (e.g. mosquito in HomeAssistant).
+`sofar_g3_lsw3_logger_reader` is a go program to integrate certain Sofar solar PV inverters with MQTT (e.g. mosquito in HomeAssistant) and/or OTLP.
 It works with Sofar inverters fitted with a LSW-3 WiFi data logger with serial number 23XXXXXXXX.
 The program:
- - loops forever, polling the modbus port on the LSW-3 and writing the data to MQTT (all local, no dependency on SolarMan cloud),
+ - loops forever, polling the modbus port on the LSW-3 and writing the data to MQTT and/or OTLP (all local, no dependency on SolarMan cloud),
  - reads its `config.yaml` from the current working directory,
  - writes logging to stderr,
  - supports MQTT Discovery.

From a2f98cbb30f7f643e8c97c24f98c36d2d71ce50c Mon Sep 17 00:00:00 2001
From: Neil Bacon <2782001+neilbacon@users.noreply.github.com>
Date: Wed, 20 Sep 2023 16:50:32 +1000
Subject: [PATCH 08/16] config

---
 .gitignore                                       |  2 ++
 .../config-example.yaml                          | 16 ++++++++++++----
 2 files changed, 14 insertions(+), 4 deletions(-)

diff --git a/.gitignore b/.gitignore
index 60c9253..b909644 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,2 +1,4 @@
 config.yaml
+sofar-x86
+sofar-arm
 .idea
diff --git a/custom_components/sofar_g3_lsw3_logger_reader/config-example.yaml b/custom_components/sofar_g3_lsw3_logger_reader/config-example.yaml
index eae62d1..83dd706 100644
--- a/custom_components/sofar_g3_lsw3_logger_reader/config-example.yaml
+++ b/custom_components/sofar_g3_lsw3_logger_reader/config-example.yaml
@@ -3,17 +3,25 @@
 inverter:
   port: 1.2.3.4:8899       # required, port name e.g. /dev/ttyUSB0 for serial or 1.2.3.4:8899 for TCP/IP
   loggerSerial: 23XXXXXXXX # required, logger serial number
-  readInterval: 60         # update interval in seconds, default 60
-  loopLogging: true        # false to avoid ever growing log file
+  readInterval: 10         # update interval in seconds, default 60
+  loopLogging: true        # false to avoid a line to log file every readInterval
   #
   # if there is a non-empty attrWhiteList then only these explicitly listed attributes are output
   attrWhiteList:
     - ActivePower_Output_Total # Total PV generation (in units of 10W)
-    - Power_PV1                # PV output of string 1 (10W)
-    - Power_PV2                # PV output of string 2 (10W); PV1 + PV2 > ActivePower_Output_Total by about 3.5% due to inverter inefficiency?
     - ActivePower_Load_Sys     # total power consumption (10W)
     - ActivePower_PCC_Total    # grid export (+) or import (-) (10W) = ActivePower_Output_Total - ActivePower_Load_Sys (so it is redundant data)
+    - Power_PV1                # PV output of string 1 (10W)
+    - Power_PV2                # PV output of string 2 (10W); PV1 + PV2 > ActivePower_Output_Total by about 3.5% due to inverter inefficiency?
     - PV_Generation_Today      # generation since midnight (10Wh)
+    - PV_Generation_Total
+    - GenerationTime_Today
+    - GenerationTime_Total
+    - Frequency_Grid
+    - Voltage_Phase_R          # grid voltage on phase R/1
+    - InsulationResistance     # changes to these last ones might indicate a problem
+    - Temperature_Env1
+    - Temperature_HeatSink1
   # else attributes are output unless they match a regex in attrBlackList
   attrBlackList:
     - "^[ST]_"                 # prefix R_, S_, T_ for 3 phases, only R_ used in single phase systems

From fd1200407b7856137c4db8f3ab688a2c1200a6ea Mon Sep 17 00:00:00 2001
From: Neil Bacon <2782001+neilbacon@users.noreply.github.com>
Date: Thu, 21 Sep 2023 15:20:24 +1000
Subject: [PATCH 09/16] doc

---
 .../sofar_g3_lsw3_logger_reader/config-example.yaml          | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/custom_components/sofar_g3_lsw3_logger_reader/config-example.yaml b/custom_components/sofar_g3_lsw3_logger_reader/config-example.yaml
index 1ba1308..f26b02c 100644
--- a/custom_components/sofar_g3_lsw3_logger_reader/config-example.yaml
+++ b/custom_components/sofar_g3_lsw3_logger_reader/config-example.yaml
@@ -3,7 +3,7 @@
 inverter:
   port: 1.2.3.4:8899       # required, port name e.g. /dev/ttyUSB0 for serial or 1.2.3.4:8899 for TCP/IP
   loggerSerial: 23XXXXXXXX # required, logger serial number
-  readInterval: 60         # update interval in seconds, default 60
+  readInterval: 10         # update interval in seconds, default 60
   loopLogging: true        # false to avoid a line to log file every readInterval
   #
   # if there is a non-empty attrWhiteList then only these explicitly listed attributes are output
@@ -22,7 +22,6 @@ inverter:
     - InsulationResistance     # changes to these last ones might indicate a problem
     - Temperature_Env1
     - Temperature_HeatSink1
-    - Temperature_HeatSink2
   # else attributes are output unless they match a regex in attrBlackList
   attrBlackList:
     - "^[ST]_"                 # prefix R_, S_, T_ for 3 phases, only R_ used in single phase systems
@@ -71,7 +70,7 @@ otlp:                     # OTLP disabled if both urls blank
 #     "SysTime_Year": 23,
 #     "Temperature_Env1": 48,                # Single Plate Ambient Temperature (label from SolarMan app)
 #     "Temperature_HeatSink1": 35,           # Radiator Temperature 1 (label from SolarMan app)
-#     "Temperature_HeatSink2": 120,          # nothing with this value in SolarMan app
+#     "Temperature_HeatSink2": 120,          # value never seems to change
 
 # # PV strings of panels:
 #     "Power_PV1": 175,                      # PV output of string 1 (10W)

From 76392841e65d8d1dcb1416bc060b5b706f49a6b4 Mon Sep 17 00:00:00 2001
From: Neil Bacon <2782001+neilbacon@users.noreply.github.com>
Date: Fri, 22 Sep 2023 18:11:57 +1000
Subject: [PATCH 10/16] doc

---
 .../sofar_g3_lsw3_logger_reader/README.md          | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/custom_components/sofar_g3_lsw3_logger_reader/README.md b/custom_components/sofar_g3_lsw3_logger_reader/README.md
index 80e907c..2bfb143 100644
--- a/custom_components/sofar_g3_lsw3_logger_reader/README.md
+++ b/custom_components/sofar_g3_lsw3_logger_reader/README.md
@@ -7,16 +7,14 @@ It runs the program with:
 
 ## Installation and setup
 
-1. First get the go program `sofar_g3_lsw3_logger_reader` working on your dev machine, so that you have a working `config.yaml` and `sofar-x86` (x86) and `sofar-arm` (arm) binary executables.
-2. Copy this directory `custom_components/sofar_g3_lsw3_logger_reader` to homeassistant's `/config/custom_components` 
+1. First get the go program `sofar_g3_lsw3_logger_reader` working on your dev machine, so that you have a working `config.yaml` and `sofar-x86` (x86) and `sofar-arm` (arm) binary executables (in this directory on your dev machine)
+2. Copy this directory `custom_components/sofar_g3_lsw3_logger_reader` from your dev machine to homeassistant's `/config/custom_components` 
 (creating `/config/custom_components/sofar_g3_lsw3_logger_reader`). 
 You can use the `Samba share` or `Advanced SSH & Web Terminal` add-ons to do this.
-3. Copy the working `config.yaml` (from step 1) to the same directory.
-4. Copy the executables `sofar` (x86) and `sofar-arm` (arm) (from step 1) to this same directory.
-5. Enable the custom component by adding a line `sofar_g3_lsw3_logger_reader:` to homeassistant's `/config/configuration.yaml`.
-6. Do a full restart of homeassistant: `Developer Tools` > `YAML` > `CHECK CONFIGURATION` then `RESTART` > `Restart Home Assistant`
-7. Check the content of `err.log` in this same directory.
-8. Add the `Inverter` device to your dashhboard: `Settings` > `Devices & Services` > `Integration` > `MQTT` > `1 device` > `ADD TO DASHBOARD`
+3. Enable the custom component by adding a line `sofar_g3_lsw3_logger_reader:` to homeassistant's `/config/configuration.yaml`.
+4. Do a full restart of homeassistant: `Developer Tools` > `YAML` > `CHECK CONFIGURATION` then `RESTART` > `Restart Home Assistant`
+5. Check the content of homeassistant's `/config/custom_components/sofar_g3_lsw3_logger_reader/err.log`
+6. Add the `Inverter` device to your dashhboard: `Settings` > `Devices & Services` > `Integration` > `MQTT` > `1 device` > `ADD TO DASHBOARD`
 
 ## To Do
 

From 95604246f575252a1d3ee2f877407f39a569c06f Mon Sep 17 00:00:00 2001
From: Neil Bacon <2782001+neilbacon@users.noreply.github.com>
Date: Thu, 28 Sep 2023 17:05:51 +1000
Subject: [PATCH 11/16] buffering was superfluous

---
 adapters/comms/tcpip/tcpip.go | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/adapters/comms/tcpip/tcpip.go b/adapters/comms/tcpip/tcpip.go
index 9206aa9..6906302 100644
--- a/adapters/comms/tcpip/tcpip.go
+++ b/adapters/comms/tcpip/tcpip.go
@@ -1,7 +1,6 @@
 package tcpip
 
 import (
-	"bufio"
 	"fmt"
 	"net"
 	"time"
@@ -51,13 +50,11 @@ func (s *tcpIpPort) Read(buf []byte) (int, error) {
 		return 0, fmt.Errorf("connection is not open")
 	}
 
-	reader := bufio.NewReader(s.conn)
-
 	if err := s.conn.SetReadDeadline(time.Now().Add(timeout)); err != nil {
 		return 0, err
 	}
 
-	return reader.Read(buf)
+	return s.conn.Read(buf)
 }
 
 func (s *tcpIpPort) Write(payload []byte) (int, error) {

From f167f09cae0b3d37df6eea3f7154d2dec3d9a042 Mon Sep 17 00:00:00 2001
From: Neil Bacon <2782001+neilbacon@users.noreply.github.com>
Date: Thu, 28 Sep 2023 17:18:38 +1000
Subject: [PATCH 12/16] read until enough bytes for a valid modbus reply

---
 adapters/devices/sofar/lsw.go | 32 +++++++++++++++-----------------
 1 file changed, 15 insertions(+), 17 deletions(-)

diff --git a/adapters/devices/sofar/lsw.go b/adapters/devices/sofar/lsw.go
index bf6ac95..f04a790 100644
--- a/adapters/devices/sofar/lsw.go
+++ b/adapters/devices/sofar/lsw.go
@@ -112,27 +112,25 @@ func readRegisterRange(rr registerRange, connPort ports.CommunicationPort, seria
 		return nil, err
 	}
 
-	// read the result
-	buf := make([]byte, 2048)
-	n, err := connPort.Read(buf)
-	if err != nil {
-		return nil, err
-	}
-
-	// truncate the buffer
-	buf = buf[:n]
-	if len(buf) < 48 {
-		// short reply
-		return nil, fmt.Errorf("short reply: %d bytes", n)
+	// read enough bytes
+	buf := []byte{}
+	for {
+		b := make([]byte, 2048)
+		n, err := connPort.Read(b)
+		if n > 0 {
+			buf = append(buf, b[:n]...)
+		}
+		if err != nil {
+			return nil, err
+		}
+		if len(buf) >= 28 && len(buf) >= 28+int(buf[27]) {
+			break
+		}
 	}
-
-	replyBytesCount := buf[27]
-
-	modbusReply := buf[28 : 28+replyBytesCount]
+	modbusReply := buf[28 : 28+buf[27]]
 
 	// shove the data into the reply
 	reply := make(map[string]interface{})
-
 	for _, f := range rr.replyFields {
 		fieldOffset := (f.register - rr.start) * 2
 

From fdb59ac1f9a72a9a8a891829942e36e3f995c6d3 Mon Sep 17 00:00:00 2001
From: Neil Bacon <2782001+neilbacon@users.noreply.github.com>
Date: Tue, 17 Oct 2023 13:06:38 +1100
Subject: [PATCH 13/16] availability on/offline

---
 adapters/export/mosquitto/mqtt.go | 26 ++++------
 go.mod                            | 16 +++---
 go.sum                            | 82 +++++++++++++++++++++++++++++++
 main.go                           | 59 +++++++++++-----------
 4 files changed, 131 insertions(+), 52 deletions(-)

diff --git a/adapters/export/mosquitto/mqtt.go b/adapters/export/mosquitto/mqtt.go
index 341ddb9..9beb630 100644
--- a/adapters/export/mosquitto/mqtt.go
+++ b/adapters/export/mosquitto/mqtt.go
@@ -107,14 +107,15 @@ func (conn *Connection) InsertDiscoveryRecord(discovery string, state string, ex
 	for _, f := range fields {
 		topic := fmt.Sprintf("%s/%s/config", discovery, f.Name)
 		json, _ := json.Marshal(map[string]interface{}{
-			"name":                f.Name,
-			"unique_id":           fmt.Sprintf("%s_%s", f.Name, uniq),
-			"device_class":        unit2DeviceClass(f.Unit),
-			"state_class":         unit2StateClass(f.Unit),
-			"state_topic":         state,
-			"unit_of_measurement": f.Unit,
-			"value_template":      fmt.Sprintf("{{ value_json.%s|int * %s }}", f.Name, f.Factor),
-			"expire_after":        expireAfter, // no messages for this long makes entity "Unavailable"
+			"name":                  f.Name,
+			"unique_id":             fmt.Sprintf("%s_%s", f.Name, uniq),
+			"device_class":          unit2DeviceClass(f.Unit),
+			"state_class":           unit2StateClass(f.Unit),
+			"state_topic":           state,
+			"unit_of_measurement":   f.Unit,
+			"value_template":        fmt.Sprintf("{{ value_json.%s|int * %s }}", f.Name, f.Factor),
+			"availability_topic":    state,
+			"availability_template": "{{ value_json.availability }}",
 			"device": map[string]interface{}{
 				"identifiers": [...]string{fmt.Sprintf("Inverter_%s", uniq)},
 				"name":        "Inverter",
@@ -125,14 +126,7 @@ func (conn *Connection) InsertDiscoveryRecord(discovery string, state string, ex
 	return nil
 }
 
-func (conn *Connection) InsertRecord(measurement map[string]interface{}) error {
-	// make a copy
-	m := make(map[string]interface{}, len(measurement))
-	for k, v := range measurement {
-		m[k] = v
-	}
-	// add LastTimestamp
-	m["LastTimestamp"] = time.Now().UnixNano() / int64(time.Millisecond)
+func (conn *Connection) InsertRecord(m map[string]interface{}) error {
 	json, _ := json.Marshal(m)
 	conn.publish(conn.state, string(json), false) // state messages should not be retained
 	return nil
diff --git a/go.mod b/go.mod
index 489087b..baf3b21 100644
--- a/go.mod
+++ b/go.mod
@@ -20,17 +20,19 @@ require (
 	github.com/creack/goselect v0.1.2 // indirect
 	github.com/go-logr/logr v1.2.3 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
-	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/gorilla/websocket v1.4.2 // indirect
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.14.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.37.0 // indirect
 	go.opentelemetry.io/otel/trace v1.14.0 // indirect
 	go.opentelemetry.io/proto/otlp v0.19.0 // indirect
-	golang.org/x/net v0.7.0 // indirect
-	golang.org/x/sys v0.5.0 // indirect
-	golang.org/x/text v0.7.0 // indirect
-	google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f // indirect
-	google.golang.org/grpc v1.53.0 // indirect
-	google.golang.org/protobuf v1.28.1 // indirect
+	golang.org/x/net v0.17.0 // indirect
+	golang.org/x/sys v0.13.0 // indirect
+	golang.org/x/text v0.13.0 // indirect
+	google.golang.org/genproto v0.0.0-20231012201019-e917dd12ba7a // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20231002182017-d307bd883b97 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20231012201019-e917dd12ba7a // indirect
+	google.golang.org/grpc v1.58.2 // indirect
+	google.golang.org/protobuf v1.31.0 // indirect
 )
diff --git a/go.sum b/go.sum
index 7fab8e7..a8a4ea0 100644
--- a/go.sum
+++ b/go.sum
@@ -37,6 +37,8 @@ github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAE
 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
 github.com/cenkalti/backoff/v4 v4.2.0 h1:HN5dHm3WBOgndBH6E8V0q2jIYIR3s9yglV8k/+MN3u4=
 github.com/cenkalti/backoff/v4 v4.2.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
+github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM=
+github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
@@ -57,6 +59,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/eclipse/paho.mqtt.golang v1.3.1 h1:6F5FYb1hxVSZS+p0ji5xBQamc5ltOolTYRy5R15uVmI=
 github.com/eclipse/paho.mqtt.golang v1.3.1/go.mod h1:eTzb4gxwwyWpqBUHGQZ4ABAV7+Jgm1PklsYT/eo8Hcc=
+github.com/eclipse/paho.mqtt.golang v1.4.3 h1:2kwcUGn8seMUfWndX0hGbvH8r7crgcJguQNCyp70xik=
+github.com/eclipse/paho.mqtt.golang v1.4.3/go.mod h1:CSYvoAlsMkhYOXh/oKyxa8EcBci6dVkLCbo5tTC1RIE=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
@@ -71,6 +75,8 @@ github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2
 github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0=
 github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ=
+github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
 github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
@@ -103,6 +109,8 @@ github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw
 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
 github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
 github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
+github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
+github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
@@ -130,9 +138,15 @@ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
 github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc=
+github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 h1:BZHcxBETFHIdVyhyEfOvn/RdU/QGdLI4y34qQGjGWO0=
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4ZsPv9hVvWI6+ch50m39Pf2Ks=
+github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 h1:YBftPWNWd4WwGqtY2yeZL2ef8rHAxPBD8KFhJpmcqms=
+github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9Kz84aCaG7AsGZnLjhHbUqwPg=
+github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.0 h1:RtRsiaGvWxcwd8y3BiRZxsylPT8hLWZ5SPcfI+3IDNk=
+github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.0/go.mod h1:TzP6duP4Py2pHLVPPQp42aoYI92+PCrVotyR5e8Vqlk=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
@@ -144,6 +158,7 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
@@ -157,11 +172,14 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
+github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
 github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 go.bug.st/serial v1.4.0 h1:IXHzPVbUBbql66lQZ1iV9LWzGXT5lh6S9gZxHK/KyQE=
 go.bug.st/serial v1.4.0/go.mod h1:z8CesKorE90Qr/oRSJiEuvzYRKol9r/anJZEb5kt304=
+go.bug.st/serial v1.6.1 h1:VSSWmUxlj1T/YlRo2J104Zv3wJFrjHIl/T3NeruWAHY=
+go.bug.st/serial v1.6.1/go.mod h1:UABfsluHAiaNI+La2iESysd9Vetq7VRdpxvjx7CmmOE=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
 go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
@@ -169,25 +187,45 @@ go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opentelemetry.io/otel v1.14.0 h1:/79Huy8wbf5DnIPhemGB+zEPVwnN6fuQybr/SRXa6hM=
 go.opentelemetry.io/otel v1.14.0/go.mod h1:o4buv+dJzx8rohcUeRmWUZhqupFvzWis188WlggnNeU=
+go.opentelemetry.io/otel v1.19.0 h1:MuS/TNf4/j4IXsZuJegVzI1cwut7Qc00344rgH7p8bs=
+go.opentelemetry.io/otel v1.19.0/go.mod h1:i0QyjOq3UPoTzff0PJB2N66fb4S0+rSbSB15/oyH9fY=
 go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.14.0 h1:/fXHZHGvro6MVqV34fJzDhi7sHGpX3Ej/Qjmfn003ho=
 go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.14.0/go.mod h1:UFG7EBMRdXyFstOwH028U0sVf+AvukSGhF0g8+dmNG8=
+go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.17.0 h1:eU0ffpYuEY7eQ75K+nKr9CI5KcY8h+GPk/9DDlEO1NI=
+go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.17.0/go.mod h1:9P5RK5JS2sjKepuCkqFwPp3etwV/57E0eigLw18Mn1k=
 go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.37.0 h1:22J9c9mxNAZugv86zhwjBnER0DbO0VVpW9Oo/j3jBBQ=
 go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.37.0/go.mod h1:QD8SSO9fgtBOvXYpcX5NXW+YnDJByTnh7a/9enQWFmw=
+go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.42.0 h1:ZtfnDL+tUrs1F0Pzfwbg2d59Gru9NCH3bgSHBM6LDwU=
+go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.42.0/go.mod h1:hG4Fj/y8TR/tlEDREo8tWstl9fO9gcFkn4xrx0Io8xU=
 go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.37.0 h1:CI6DSdsSkJxX1rsfPSQ0SciKx6klhdDRBXqKb+FwXG8=
 go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.37.0/go.mod h1:WLBYPrz8srktckhCjFaau4VHSfGaMuqoKSXwpzaiRZg=
+go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.42.0 h1:NmnYCiR0qNufkldjVvyQfZTHSdzeHoZ41zggMsdMcLM=
+go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.42.0/go.mod h1:UVAO61+umUsHLtYb8KXXRoHtxUkdOPkYidzW3gipRLQ=
 go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.37.0 h1:Ad4fpLq5t4s4+xB0chYBmbp1NNMqG4QRkseRmbx3bOw=
 go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.37.0/go.mod h1:hgpB6JpYB/K403Z2wCxtX5fENB1D4bSdAHG0vJI+Koc=
+go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.42.0 h1:wNMDy/LVGLj2h3p6zg4d0gypKfWKSWI14E1C4smOgl8=
+go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.42.0/go.mod h1:YfbDdXAAkemWJK3H/DshvlrxqFB2rtW4rY6ky/3x/H0=
 go.opentelemetry.io/otel/metric v0.37.0 h1:pHDQuLQOZwYD+Km0eb657A25NaRzy0a+eLyKfDXedEs=
 go.opentelemetry.io/otel/metric v0.37.0/go.mod h1:DmdaHfGt54iV6UKxsV9slj2bBRJcKC1B1uvDLIioc1s=
+go.opentelemetry.io/otel/metric v1.19.0 h1:aTzpGtV0ar9wlV4Sna9sdJyII5jTVJEvKETPiOKwvpE=
+go.opentelemetry.io/otel/metric v1.19.0/go.mod h1:L5rUsV9kM1IxCj1MmSdS+JQAcVm319EUrDVLrt7jqt8=
 go.opentelemetry.io/otel/sdk v1.14.0 h1:PDCppFRDq8A1jL9v6KMI6dYesaq+DFcDZvjsoGvxGzY=
 go.opentelemetry.io/otel/sdk v1.14.0/go.mod h1:bwIC5TjrNG6QDCHNWvW4HLHtUQ4I+VQDsnjhvyZCALM=
+go.opentelemetry.io/otel/sdk v1.19.0 h1:6USY6zH+L8uMH8L3t1enZPR3WFEmSTADlqldyHtJi3o=
+go.opentelemetry.io/otel/sdk v1.19.0/go.mod h1:NedEbbS4w3C6zElbLdPJKOpJQOrGUJ+GfzpjUvI0v1A=
 go.opentelemetry.io/otel/sdk/metric v0.37.0 h1:haYBBtZZxiI3ROwSmkZnI+d0+AVzBWeviuYQDeBWosU=
 go.opentelemetry.io/otel/sdk/metric v0.37.0/go.mod h1:mO2WV1AZKKwhwHTV3AKOoIEb9LbUaENZDuGUQd+j4A0=
+go.opentelemetry.io/otel/sdk/metric v1.19.0 h1:EJoTO5qysMsYCa+w4UghwFV/ptQgqSL/8Ni+hx+8i1k=
+go.opentelemetry.io/otel/sdk/metric v1.19.0/go.mod h1:XjG0jQyFJrv2PbMvwND7LwCEhsJzCzV5210euduKcKY=
 go.opentelemetry.io/otel/trace v1.14.0 h1:wp2Mmvj41tDsyAJXiWDWpfNsOiIyd38fy85pyKcFq/M=
 go.opentelemetry.io/otel/trace v1.14.0/go.mod h1:8avnQLK+CG77yNLUae4ea2JDQ6iT+gozhnZjy/rw9G8=
+go.opentelemetry.io/otel/trace v1.19.0 h1:DFVQmlVbfVeOuBRrwdtaehRrWiL1JoVs9CPIQ1Dzxpg=
+go.opentelemetry.io/otel/trace v1.19.0/go.mod h1:mfaSyvGyEJEI0nyV2I4qhNQnbBOUUmYZpYojqMnX2vo=
 go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
 go.opentelemetry.io/proto/otlp v0.19.0 h1:IVN6GR+mhC4s5yfcTbmzHYODqvWAp3ZedA2SJPI1Nnw=
 go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U=
+go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I=
+go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
@@ -255,6 +293,12 @@ golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81R
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
 golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g=
 golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M=
+golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
+golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50=
+golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA=
+golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
+golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -269,6 +313,10 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E=
+golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
+golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ=
+golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -299,8 +347,13 @@ golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM=
 golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o=
+golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
+golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -310,6 +363,12 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE=
+golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
+golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4=
+golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
+golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
+golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -412,6 +471,22 @@ google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6D
 google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
 google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f h1:BWUVssLB0HVOSY78gIdvk1dTVYtT1y8SBWtPYuTJ/6w=
 google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM=
+google.golang.org/genproto v0.0.0-20230711160842-782d3b101e98 h1:Z0hjGZePRE0ZBWotvtrwxFNrNE9CUAGtplaDK5NNI/g=
+google.golang.org/genproto v0.0.0-20230711160842-782d3b101e98/go.mod h1:S7mY02OqCJTD0E1OiQy1F72PWFB4bZJ87cAtLPYgDR0=
+google.golang.org/genproto v0.0.0-20231002182017-d307bd883b97 h1:SeZZZx0cP0fqUyA+oRzP9k7cSwJlvDFiROO72uwD6i0=
+google.golang.org/genproto v0.0.0-20231002182017-d307bd883b97/go.mod h1:t1VqOqqvce95G3hIDCT5FeO3YUc6Q4Oe24L/+rNMxRk=
+google.golang.org/genproto v0.0.0-20231012201019-e917dd12ba7a h1:fwgW9j3vHirt4ObdHoYNwuO24BEZjSzbh+zPaNWoiY8=
+google.golang.org/genproto v0.0.0-20231012201019-e917dd12ba7a/go.mod h1:EMfReVxb80Dq1hhioy0sOsY9jCE46YDgHlJ7fWVUWRE=
+google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98 h1:FmF5cCW94Ij59cfpoLiwTgodWmm60eEV0CjlsVg2fuw=
+google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98/go.mod h1:rsr7RhLuwsDKL7RmgDDCUc6yaGr1iqceVb5Wv6f6YvQ=
+google.golang.org/genproto/googleapis/api v0.0.0-20231002182017-d307bd883b97 h1:W18sezcAYs+3tDZX4F80yctqa12jcP1PUS2gQu1zTPU=
+google.golang.org/genproto/googleapis/api v0.0.0-20231002182017-d307bd883b97/go.mod h1:iargEX0SFPm3xcfMI0d1domjg0ZF4Aa0p2awqyxhvF0=
+google.golang.org/genproto/googleapis/api v0.0.0-20231012201019-e917dd12ba7a h1:myvhA4is3vrit1a6NZCWBIwN0kNEnX21DJOJX/NvIfI=
+google.golang.org/genproto/googleapis/api v0.0.0-20231012201019-e917dd12ba7a/go.mod h1:SUBoKXbI1Efip18FClrQVGjWcyd0QZd8KkvdP34t7ww=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 h1:bVf09lpb+OJbByTj913DRJioFFAjf/ZGxEz7MajTp2U=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98/go.mod h1:TUfxEVdsvPg18p6AslUXFoLdpED4oBnGwyqk3dV1XzM=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20231012201019-e917dd12ba7a h1:a2MQQVoTo96JC9PMGtGBymLp7+/RzpFc2yX/9WfFg1c=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20231012201019-e917dd12ba7a/go.mod h1:4cYg8o5yUbm77w8ZX00LhMVNl/YVBFJRYWDc0uYWMs0=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
 google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
@@ -430,6 +505,11 @@ google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9K
 google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU=
 google.golang.org/grpc v1.53.0 h1:LAv2ds7cmFV/XTS3XG1NneeENYrXGmorPxsBbptIjNc=
 google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw=
+google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo=
+google.golang.org/grpc v1.58.2 h1:SXUpjxeVF3FKrTYQI4f4KvbGD5u2xccdYdurwowix5I=
+google.golang.org/grpc v1.58.2/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0=
+google.golang.org/grpc v1.58.3 h1:BjnpXut1btbtgN/6sp+brB2Kbm2LjNXnidYujAVbSoQ=
+google.golang.org/grpc v1.58.3/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
 google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
@@ -445,6 +525,8 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ
 google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w=
 google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8=
+google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
diff --git a/main.go b/main.go
index db1c1ef..9462801 100644
--- a/main.go
+++ b/main.go
@@ -17,8 +17,6 @@ import (
 	"github.com/kubaceg/sofar_g3_lsw3_logger_reader/ports"
 )
 
-// maximumFailedConnections maximum number failed logger connection, after this number will be exceeded reconnect
-// interval will be extended from 5s to readInterval defined in config file
 const maximumFailedConnections = 3
 
 var (
@@ -75,51 +73,54 @@ func main() {
 		mqtt.InsertDiscoveryRecord(config.Mqtt.Discovery, config.Mqtt.State, config.Inverter.ReadInterval*5, device.GetDiscoveryFields())
 	}
 
-	failedConnections := 0
-
 	for {
 		if config.Inverter.LoopLogging {
 			log.Printf("performing measurements")
 		}
-		timeStart := time.Now()
-
-		measurements, err := device.Query()
-		if err != nil {
-			log.Printf("failed to perform measurements: %s", err)
-			// at night, inverter is offline, err = "dial tcp 192.168.xx.xxx:8899: i/o timeout"
-			// At other times we occaisionally get this error and also: "short reply: xx bytes"
-			failedConnections++
 
-			if failedConnections > maximumFailedConnections {
-				time.Sleep(time.Duration(config.Inverter.ReadInterval) * time.Second)
+		var measurements map[string]interface{} = nil
+		var err error
+		for retry := 0; measurements == nil && retry < maximumFailedConnections; retry++ {
+			measurements, err = device.Query()
+			if err != nil {
+				log.Printf("failed to perform measurements on retry %d: %s", retry, err)
+				// at night, inverter is offline, err = "dial tcp 192.168.xx.xxx:8899: i/o timeout"
+				// at other times occaisionally: "read tcp 192.168.68.104:38670->192.168.68.106:8899: i/o timeout"
 			}
-
-			continue
 		}
 
-		failedConnections = 0
-
 		if hasMQTT {
-			// removed from async go func 'goroutine', not needed and proper usage requires WaitGroup to wait for completion
-			mqtt.InsertRecord(measurements) // logs errors, always returns nil
+			var m map[string]interface{} = nil
+			timeStamp := time.Now().UnixNano() / int64(time.Millisecond)
+			if measurements != nil {
+				m = make(map[string]interface{}, len(measurements)+2)
+				for k, v := range measurements {
+					m[k] = v
+				}
+				m["availability"] = "online"
+				m["LastTimestamp"] = timeStamp
+			} else {
+				m = map[string]interface{}{
+					"availability":  "offline",
+					"LastTimestamp": timeStamp,
+				}
+			}
+			err := mqtt.InsertRecord(m) // logs errors, always returns nil
+			if err != nil {
+			}
 		}
 
-		if hasOTLP {
-			// removed from async go func 'goroutine'
-			err = telem.CollectAndPushMetrics(context.Background(), measurements)
+		if hasOTLP && measurements != nil {
+			err := telem.CollectAndPushMetrics(context.Background(), measurements)
 			if err != nil {
 				log.Printf("error recording telemetry: %s\n", err)
 			} else {
 				log.Println("measurements pushed via OLTP")
 			}
-		}
 
-		// if mqtt & otlp were done async then the WaitGroup to wait for completion would go here
-		duration := time.Since(timeStart)
-		delay := time.Duration(config.Inverter.ReadInterval)*time.Second - duration
-		if delay > 0 {
-			time.Sleep(delay)
 		}
+
+		time.Sleep(time.Duration(config.Inverter.ReadInterval) * time.Second)
 	}
 
 }

From b27d24774db2c2a892f089efb483fca7cdcc10f5 Mon Sep 17 00:00:00 2001
From: Neil Bacon <2782001+neilbacon@users.noreply.github.com>
Date: Sun, 22 Oct 2023 14:47:48 +1100
Subject: [PATCH 14/16] address linter warning

---
 .gitignore | 1 +
 main.go    | 8 ++++++--
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/.gitignore b/.gitignore
index b909644..d921623 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,4 +1,5 @@
 config.yaml
 sofar-x86
 sofar-arm
+.vscode
 .idea
diff --git a/main.go b/main.go
index 9462801..693a964 100644
--- a/main.go
+++ b/main.go
@@ -70,7 +70,10 @@ func main() {
 	initialize()
 
 	if hasMQTT {
-		mqtt.InsertDiscoveryRecord(config.Mqtt.Discovery, config.Mqtt.State, config.Inverter.ReadInterval*5, device.GetDiscoveryFields())
+		err := mqtt.InsertDiscoveryRecord(config.Mqtt.Discovery, config.Mqtt.State, config.Inverter.ReadInterval*5, device.GetDiscoveryFields()) // logs errors, always returns nil
+		if err != nil {
+			log.Printf("never happens: %s", err)
+		}
 	}
 
 	for {
@@ -90,7 +93,7 @@ func main() {
 		}
 
 		if hasMQTT {
-			var m map[string]interface{} = nil
+			var m map[string]interface{}
 			timeStamp := time.Now().UnixNano() / int64(time.Millisecond)
 			if measurements != nil {
 				m = make(map[string]interface{}, len(measurements)+2)
@@ -107,6 +110,7 @@ func main() {
 			}
 			err := mqtt.InsertRecord(m) // logs errors, always returns nil
 			if err != nil {
+				log.Printf("never happens: %s", err)
 			}
 		}
 

From d16467368bf0a97e3d2fe1108d947948e9e9de24 Mon Sep 17 00:00:00 2001
From: Neil Bacon <2782001+neilbacon@users.noreply.github.com>
Date: Sun, 22 Oct 2023 15:06:41 +1100
Subject: [PATCH 15/16] delete custom_component & updated README

---
 README.md                                     | 43 +++------
 .../sofar_g3_lsw3_logger_reader/README.md     | 23 -----
 .../sofar_g3_lsw3_logger_reader/__init__.py   | 26 ------
 .../config-example.yaml                       | 90 -------------------
 .../sofar_g3_lsw3_logger_reader/manifest.json |  9 --
 5 files changed, 13 insertions(+), 178 deletions(-)
 delete mode 100644 custom_components/sofar_g3_lsw3_logger_reader/README.md
 delete mode 100644 custom_components/sofar_g3_lsw3_logger_reader/__init__.py
 delete mode 100644 custom_components/sofar_g3_lsw3_logger_reader/config-example.yaml
 delete mode 100644 custom_components/sofar_g3_lsw3_logger_reader/manifest.json

diff --git a/README.md b/README.md
index 8abea79..bbef8ae 100644
--- a/README.md
+++ b/README.md
@@ -1,41 +1,24 @@
 # Sofar g3 LSW-3 logger reader
-
-`sofar_g3_lsw3_logger_reader` is a go program to integrate certain Sofar solar PV inverters with MQTT (e.g. mosquito in HomeAssistant) and/or OTLP.
-It works with Sofar inverters fitted with a LSW-3 WiFi data logger with serial number 23XXXXXXXX.
-The program:
- - loops forever, polling the modbus port on the LSW-3 and writing the data to MQTT and/or OTLP (all local, no dependency on SolarMan cloud),
- - reads its `config.yaml` from the current working directory,
- - writes logging to stderr,
- - supports MQTT Discovery.
-
-`custom_components/sofar_g3_lsw3_logger_reader` is a HomeAssistant custom component to run the above program.
+Tool written in GO for reading metrics from Sofar LSW-3 and writing results into MQTT topics. 
+Program queries logger modbus port in infinite loop and sends data into MQTT topics (e.g. mosquito in HomeAssistant).
 
 ## Installation and setup
-
-1. Download go 1.19 - 1.21
-2. Clone this repo `git clone git@github.com:neilbacon/sofar_g3_lsw3_logger_reader.git`
+1. Download go 1.19
+2. Clone this repo `git clone git@github.com:kubaceg/sofar_g3_lsw3_logger_reader.git`
 3. Go into project directory `cd sofar_g3_lsw3_logger_reader`
-6. Build program `make` (creates binaries in `custom_components/sofar_g3_lsw3_logger_reader`)
-4. Go into directory `custom_components/sofar_g3_lsw3_logger_reader` and copy example config `cp config-example.yaml config.yaml` and edit `config.yaml` to suit your needs
-7. Run `./sofar-x86` on x86 or `./sofar-arm` on arm (including Raspberry Pi)
-
-To run it as part of Home Assistant see the [custom component README](custom_components/sofar_g3_lsw3_logger_reader/README.md).
+4. Copy example config `cp config-example.yaml config.yaml`
+5. Edit `config.yaml` in Your favorite editor, fill all required stuff
+6. Build program `make build` or build for ARM machines e.g. raspberryPi `make build-arm`
+7. Run `./sofar` or `sofar-arm`
 
 ## Output data format
-
 ### MQTT
+Data will be sent into MQTT topic with name `{mqttPrefix}/{fieldName}` where:
+* mqttPrefix is value defined in `config.yaml` e.g. `/sensors/energy/inverter`
+* fieldName is measurement name, all available measurements are described in `adapters/devices/sofar/sofar_protocol.go`, e.g. `PV_Generation_Today`
 
-#### Attribute Filtering
-The LSW-3 provides a large number of attributes that you are most likely not interested in, so `config.yaml` allows you to filter them using either a white list or a black list used only if the white list is empty. The white list contains the attribute names (in full) to include and the black list contains regular expressions for attributes to exclude.
-
-#### MQTT Discovery
-On startup a message is sent (with `retain=true`) on a configuration topic for each data attribute. Home Assistant uses this information to configure an entity to manage the data attribute, removing the need for much manual configuration. The topic used is `{mqtt.discovery}/{attribute}/config` where `{mqtt.discovery}` comes from `config.yaml` and `{attribute}` is the attribute name. The JSON payload is described in the MQTT Discovery documentation.
-
-`retain=true` causes a newly connected client (such as a restarting Home Assistant) to receive a copy of these messages. MQTT Discovery documentation suggests this is a good idea, but in development it can be a pain, leaving old messages hanging around indefinitely. You can use the `sh/cleanmqtt.sh` utility to clean up these messages. It depends on a MQTT installation and can be run in the HA OS's mosquitto container `addon_core_mosquitto`.
-
-#### MQTT Data
-
-All attributes are sent in a single message (with retain=false) with JSON payload to the topic `{mqtt.state}` specified in `config.yaml`.
+Full topic name for given example values is `/sensors/energy/inverter/PV_Generation_Today`.
+Additional field is `All` which contains all measurements and their values marshalled into one json.
 
 ### OTLP
 Data can also be sent over OTLP protocol to a gRPC or http server. Typically, this would be received by the 
diff --git a/custom_components/sofar_g3_lsw3_logger_reader/README.md b/custom_components/sofar_g3_lsw3_logger_reader/README.md
deleted file mode 100644
index 2bfb143..0000000
--- a/custom_components/sofar_g3_lsw3_logger_reader/README.md
+++ /dev/null
@@ -1,23 +0,0 @@
-# Sofar g3 LSW-3 logger reader Custom Component
-
-This is a HomeAssistant custom component to run the [sofar_g3_lsw3_logger_reader](../../README.md) program.
-It runs the program with:
-  - the current working directory set to /config/custom_components/sofar_g3_lsw3_logger_reader
-  - stdout/err going to files out/err.log (because I haven't succeeded in integrating it with homeassistant logging)
-
-## Installation and setup
-
-1. First get the go program `sofar_g3_lsw3_logger_reader` working on your dev machine, so that you have a working `config.yaml` and `sofar-x86` (x86) and `sofar-arm` (arm) binary executables (in this directory on your dev machine)
-2. Copy this directory `custom_components/sofar_g3_lsw3_logger_reader` from your dev machine to homeassistant's `/config/custom_components` 
-(creating `/config/custom_components/sofar_g3_lsw3_logger_reader`). 
-You can use the `Samba share` or `Advanced SSH & Web Terminal` add-ons to do this.
-3. Enable the custom component by adding a line `sofar_g3_lsw3_logger_reader:` to homeassistant's `/config/configuration.yaml`.
-4. Do a full restart of homeassistant: `Developer Tools` > `YAML` > `CHECK CONFIGURATION` then `RESTART` > `Restart Home Assistant`
-5. Check the content of homeassistant's `/config/custom_components/sofar_g3_lsw3_logger_reader/err.log`
-6. Add the `Inverter` device to your dashhboard: `Settings` > `Devices & Services` > `Integration` > `MQTT` > `1 device` > `ADD TO DASHBOARD`
-
-## To Do
-
-1. Add to HACS
-2. Get logging going to homeassistant's log.
-3. Add support for "config flow" 
diff --git a/custom_components/sofar_g3_lsw3_logger_reader/__init__.py b/custom_components/sofar_g3_lsw3_logger_reader/__init__.py
deleted file mode 100644
index c5d38ec..0000000
--- a/custom_components/sofar_g3_lsw3_logger_reader/__init__.py
+++ /dev/null
@@ -1,26 +0,0 @@
-"""Integration for Sofar Solar PV inverters with LSW-3 WiFi sticks with SN 23XXXXXXX"""
-
-import platform
-import subprocess
-import logging
-import os
-
-_LOGGER = logging.getLogger(__name__)
-_DOMAIN = "sofar_g3_lsw3_logger_reader"
-_DIR = f"/config/custom_components/{_DOMAIN}"
-
-def setup(hass, config):
-    hass.states.set(f"{_DOMAIN}.status", __name__)                  # custom_components.sofar_g3_lsw3_logger_reader
-    _LOGGER.warning("setup: Working Dir is %s", os.getcwd())        # /config
-    _LOGGER.info("setup: Working Dir is %s", os.getcwd())           # main configuration.yaml should enable info, but this is not yet working  
-
-    # When the process writes a line to stderr, I'd like it to go to _LOGGER.info(), but I haven't fugured out how to do this
-    # without blocking, so for now we connect stdout/err to log files.
-    out = open(f"{_DIR}/out.log", "w")                              # empty
-    err = open(f"{_DIR}/err.log", "w")                              # go logging package writes to stderr
-    # platform.processor() is '' on HA OS on Raspberry Pi
-    exe = f"{_DIR}/sofar-x86" if platform.processor().startswith("x86") else f"{_DIR}/sofar-arm" 
-    proc = subprocess.Popen(exe, cwd=f"{_DIR}", stdout=out, stderr=err)
-    # the (go lang) sofar process reads its config.yaml from cwd, then loops forever
-    _LOGGER.warning("setup: end")
-    return True
diff --git a/custom_components/sofar_g3_lsw3_logger_reader/config-example.yaml b/custom_components/sofar_g3_lsw3_logger_reader/config-example.yaml
deleted file mode 100644
index f26b02c..0000000
--- a/custom_components/sofar_g3_lsw3_logger_reader/config-example.yaml
+++ /dev/null
@@ -1,90 +0,0 @@
-# make your own in config.yaml
-
-inverter:
-  port: 1.2.3.4:8899       # required, port name e.g. /dev/ttyUSB0 for serial or 1.2.3.4:8899 for TCP/IP
-  loggerSerial: 23XXXXXXXX # required, logger serial number
-  readInterval: 10         # update interval in seconds, default 60
-  loopLogging: true        # false to avoid a line to log file every readInterval
-  #
-  # if there is a non-empty attrWhiteList then only these explicitly listed attributes are output
-  attrWhiteList:
-    - ActivePower_Output_Total # Total PV generation (in units of 10W)
-    - ActivePower_Load_Sys     # total power consumption (10W)
-    - ActivePower_PCC_Total    # grid export (+) or import (-) (10W) = ActivePower_Output_Total - ActivePower_Load_Sys (so it is redundant data)
-    - Power_PV1                # PV output of string 1 (10W)
-    - Power_PV2                # PV output of string 2 (10W); PV1 + PV2 > ActivePower_Output_Total by about 3.5% due to inverter inefficiency?
-    - PV_Generation_Today      # generation since midnight (10Wh)
-    - PV_Generation_Total
-    - GenerationTime_Today
-    - GenerationTime_Total
-    - Frequency_Grid
-    - Voltage_Phase_R          # grid voltage on phase R/1
-    - InsulationResistance     # changes to these last ones might indicate a problem
-    - Temperature_Env1
-    - Temperature_HeatSink1
-  # else attributes are output unless they match a regex in attrBlackList
-  attrBlackList:
-    - "^[ST]_"                 # prefix R_, S_, T_ for 3 phases, only R_ used in single phase systems
-    - "_[ST]$"                 # likewise suffixes _R, _S, _T
-
-mqtt:                     # MQTT disabled if url & prefix both blank
-  url: 1.2.3.4:1883       # MQTT broker URL (e.g. 1.2.3.4:1883)
-  user:                   # MQTT username (leave empty when not needed)
-  password:               # MQTT password (leave empty when not needed)
-  discovery: homeassistant/sensor   # topic prefix for MQTT Discovery
-  state:     energy/inverter/state  # topic for state
-
-otlp:                     # OTLP disabled if both urls blank
-  grpc:
-    url:                  # URL for gRPC OTLP server e.g. 0.0.0.0:4317
-  http:
-    url:                  # URL for HTTP OLTP server e.g. 0.0.0.0:4318 
-  prefix: sofar.logging
-
-# List of all attributes with non-zero values fetched from my inverter:
-#
-# # Most interesting to me:
-#     "ActivePower_Load_Sys": 16,        # total power consumption (10W)
-#     "ActivePower_Output_Total": 333,   # Total PV generation (in units of 10W)
-#     "ActivePower_PCC_Total": 317,      # grid export (+) or import (-) (10W) = ActivePower_Output_Total - ActivePower_Load_Sys (so it is redundant data)
-#     "GenerationTime_Today": 553,
-#     "GenerationTime_Total": 54992,
-#     "PV_Generation_Today": 3428,       # generation since midnight (10Wh)
-#     "PV_Generation_Total": 20355,
-
-# # Not credible:
-#     "Load_Consumption_Today": 3428,    # same as PV_Generation_Today, so not credible as Load
-#     "Load_Consumption_Total": 20355,   # same as PV_Generation_Total, so not credible as Load
-
-# # Inverter:
-#     "ApparentPower_Output_Total": 33,
-#     "Countdown": 60,
-#     "InsulationResistance": 1437,
-#     "ServiceTime_Total": 56198,
-#     "SysState": 2,
-#     "SysTime_Date": 19,
-#     "SysTime_Hour": 14,
-#     "SysTime_Minute": 56,
-#     "SysTime_Month": 9,
-#     "SysTime_Second": 1,
-#     "SysTime_Year": 23,
-#     "Temperature_Env1": 48,                # Single Plate Ambient Temperature (label from SolarMan app)
-#     "Temperature_HeatSink1": 35,           # Radiator Temperature 1 (label from SolarMan app)
-#     "Temperature_HeatSink2": 120,          # value never seems to change
-
-# # PV strings of panels:
-#     "Power_PV1": 175,                      # PV output of string 1 (10W)
-#     "Power_PV2": 172,                      # PV output of string 2 (10W); PV1 + PV2 > ActivePower_Output_Total by about 3.5% due to inverter inefficiency?
-#     "Voltage_PV1": 2438,
-#     "Voltage_PV2": 2436,
-#     "Current_PV1": 720,
-#     "Current_PV2": 706,
-
-# # Grid:
-#     "Frequency_Grid": 5000,
-
-# # Phase R (single phase power):
-#     "ActivePower_PCC_R": 317,              # same as ActivePower_PCC_Total
-#     "Voltage_Phase_R": 2352                # grid voltage
-#     "Current_Output_R": 1418,
-#     "Current_PCC_R": 1394,
diff --git a/custom_components/sofar_g3_lsw3_logger_reader/manifest.json b/custom_components/sofar_g3_lsw3_logger_reader/manifest.json
deleted file mode 100644
index 1033cf0..0000000
--- a/custom_components/sofar_g3_lsw3_logger_reader/manifest.json
+++ /dev/null
@@ -1,9 +0,0 @@
-{
-  "domain": "sofar_g3_lsw3_logger_reader",
-  "name": "Sofar LSW3",
-  "version": "0.1.0",
-  "integration_type": "device",
-  "iot_class": "local_push",
-  "dependencies": ["mqtt"],
-  "documentation": "https://github.com/neilbacon/sofar_g3_lsw3_logger_reader"
-}

From 8eb66a33afd7524a6474be599cbef0e76e9ec8a1 Mon Sep 17 00:00:00 2001
From: Neil Bacon <2782001+neilbacon@users.noreply.github.com>
Date: Sun, 22 Oct 2023 15:09:48 +1100
Subject: [PATCH 16/16] config-example.yaml

---
 config-example.yaml | 90 +++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 90 insertions(+)
 create mode 100644 config-example.yaml

diff --git a/config-example.yaml b/config-example.yaml
new file mode 100644
index 0000000..f26b02c
--- /dev/null
+++ b/config-example.yaml
@@ -0,0 +1,90 @@
+# make your own in config.yaml
+
+inverter:
+  port: 1.2.3.4:8899       # required, port name e.g. /dev/ttyUSB0 for serial or 1.2.3.4:8899 for TCP/IP
+  loggerSerial: 23XXXXXXXX # required, logger serial number
+  readInterval: 10         # update interval in seconds, default 60
+  loopLogging: true        # false to avoid a line to log file every readInterval
+  #
+  # if there is a non-empty attrWhiteList then only these explicitly listed attributes are output
+  attrWhiteList:
+    - ActivePower_Output_Total # Total PV generation (in units of 10W)
+    - ActivePower_Load_Sys     # total power consumption (10W)
+    - ActivePower_PCC_Total    # grid export (+) or import (-) (10W) = ActivePower_Output_Total - ActivePower_Load_Sys (so it is redundant data)
+    - Power_PV1                # PV output of string 1 (10W)
+    - Power_PV2                # PV output of string 2 (10W); PV1 + PV2 > ActivePower_Output_Total by about 3.5% due to inverter inefficiency?
+    - PV_Generation_Today      # generation since midnight (10Wh)
+    - PV_Generation_Total
+    - GenerationTime_Today
+    - GenerationTime_Total
+    - Frequency_Grid
+    - Voltage_Phase_R          # grid voltage on phase R/1
+    - InsulationResistance     # changes to these last ones might indicate a problem
+    - Temperature_Env1
+    - Temperature_HeatSink1
+  # else attributes are output unless they match a regex in attrBlackList
+  attrBlackList:
+    - "^[ST]_"                 # prefix R_, S_, T_ for 3 phases, only R_ used in single phase systems
+    - "_[ST]$"                 # likewise suffixes _R, _S, _T
+
+mqtt:                     # MQTT disabled if url & prefix both blank
+  url: 1.2.3.4:1883       # MQTT broker URL (e.g. 1.2.3.4:1883)
+  user:                   # MQTT username (leave empty when not needed)
+  password:               # MQTT password (leave empty when not needed)
+  discovery: homeassistant/sensor   # topic prefix for MQTT Discovery
+  state:     energy/inverter/state  # topic for state
+
+otlp:                     # OTLP disabled if both urls blank
+  grpc:
+    url:                  # URL for gRPC OTLP server e.g. 0.0.0.0:4317
+  http:
+    url:                  # URL for HTTP OLTP server e.g. 0.0.0.0:4318 
+  prefix: sofar.logging
+
+# List of all attributes with non-zero values fetched from my inverter:
+#
+# # Most interesting to me:
+#     "ActivePower_Load_Sys": 16,        # total power consumption (10W)
+#     "ActivePower_Output_Total": 333,   # Total PV generation (in units of 10W)
+#     "ActivePower_PCC_Total": 317,      # grid export (+) or import (-) (10W) = ActivePower_Output_Total - ActivePower_Load_Sys (so it is redundant data)
+#     "GenerationTime_Today": 553,
+#     "GenerationTime_Total": 54992,
+#     "PV_Generation_Today": 3428,       # generation since midnight (10Wh)
+#     "PV_Generation_Total": 20355,
+
+# # Not credible:
+#     "Load_Consumption_Today": 3428,    # same as PV_Generation_Today, so not credible as Load
+#     "Load_Consumption_Total": 20355,   # same as PV_Generation_Total, so not credible as Load
+
+# # Inverter:
+#     "ApparentPower_Output_Total": 33,
+#     "Countdown": 60,
+#     "InsulationResistance": 1437,
+#     "ServiceTime_Total": 56198,
+#     "SysState": 2,
+#     "SysTime_Date": 19,
+#     "SysTime_Hour": 14,
+#     "SysTime_Minute": 56,
+#     "SysTime_Month": 9,
+#     "SysTime_Second": 1,
+#     "SysTime_Year": 23,
+#     "Temperature_Env1": 48,                # Single Plate Ambient Temperature (label from SolarMan app)
+#     "Temperature_HeatSink1": 35,           # Radiator Temperature 1 (label from SolarMan app)
+#     "Temperature_HeatSink2": 120,          # value never seems to change
+
+# # PV strings of panels:
+#     "Power_PV1": 175,                      # PV output of string 1 (10W)
+#     "Power_PV2": 172,                      # PV output of string 2 (10W); PV1 + PV2 > ActivePower_Output_Total by about 3.5% due to inverter inefficiency?
+#     "Voltage_PV1": 2438,
+#     "Voltage_PV2": 2436,
+#     "Current_PV1": 720,
+#     "Current_PV2": 706,
+
+# # Grid:
+#     "Frequency_Grid": 5000,
+
+# # Phase R (single phase power):
+#     "ActivePower_PCC_R": 317,              # same as ActivePower_PCC_Total
+#     "Voltage_Phase_R": 2352                # grid voltage
+#     "Current_Output_R": 1418,
+#     "Current_PCC_R": 1394,