v0.2.0, Improved register detection, improved performance, MQTT and PVOutput rewrites

SunGather/config-example.yaml

@@ -1,41 +1,43 @@
 inverter:
-  host: 192.168.1.100     # [Required] IP Address of the Inverter or Dongle
+  host: 192.168.1.100                       # [Required] IP Address of the Inverter or Dongle
-  # port: 502             # [Optional] Default for modbus is 502, for http is 8082
+  # port: 502                               # [Optional] Default for modbus is 502, for http is 8082
-  # slave: 0x01           # [Optional] Default is 0x01
+  # slave: 0x01                             # [Optional] Default is 0x01
-  # timeout: 10           # [Optional] Default is 10
+  # timeout: 10                             # [Optional] Default is 10
-  # scan_interval: 30     # [Optional] Default is 30
+  # scan_interval: 30                       # [Optional] Default is 30
-  connection: sungrow     # [Required] options: modbus, sungrow, http
+  connection: sungrow                       # [Required] options: modbus, sungrow, http
-  # model: "SG7.0RT"      # [Optional] This is autodetected on startup, only needed if detection issues or for testing
+  # model: "SG7.0RT"                        # [Optional] This is autodetected on startup, only needed if detection issues or for testing
-                          # See model list here: https://github.com/bohdan-s/SunGather#supported
+                                            # See model list here: https://github.com/bohdan-s/SunGather#supported
-  # smart_meter: True     # [Optional] Default is False, Set to true if inverter supports reading grind / house consumption
+  # smart_meter: True                       # [Optional] Default is False, Set to true if inverter supports reading grind / house consumption
-  # use_local_time: False # [Optional] Default False, Uses Inventer time, if try uses PC time when updating timestamps (e.g. PVOutput)
+  # use_local_time: False                   # [Optional] Default False, Uses Inventer time, if try uses PC time when updating timestamps (e.g. PVOutput)
-  # hybrid: False         # [Optional] Default false, if you have a Hybrid (battery) set to True
+  # manual_load: False                      # [Optional] Manually calculate load total_active_power + meter_power if the inverter does not supply it
-  # manual_load: False    # [Optional] Manually calculate load total_active_power + meter_power if the inverter does not supply it
+  # logging: 30                             # [Optional] 10 = Debug, 20 = Info, 30 = Warning (default), 40 = Error
-  # logging: 30           # [Optional] 10 = Debug, 20 = Info, 30 = Warning (default), 40 = Error
+  # level: 1                                # [Optional] Set the amount of information to gather
-  # level: 1              # [Optional] Set the amount of information to gather
+                                            # 0 = Model and Solar Generation, 
-                          # 0 = Model and Solar Generation, 
+                                            # 1 (default) = Useful data, all required for exports, 
-                          # 1 (default) = Useful data, all required for exports, 
+                                            # 2 everything your Inverter supports, 
-                          # 2 everything your Inverter supports, 
+                                            # 3 Everything from every register 
-                          # 3 Everything from every register 
 
+# If you do not want to use a export, you can either remove the whole configuration block
+# or set enabled: False
 exports:
   # Print Registers to console, good for debugging / troubleshooting
   - name: console         
-    enabled: False
+    enabled: False                          # [Optional] Default is False
 
   # Runs a simple Webserver showing Config and last read registers
-  - name: http 
+  # Access at http://localhost:8080 or http://[serverip]:8080
-    enabled: True   # Access at http://localhost:8080 or http://[serverip]:8080
+  - name: webserver 
-    # port: 8080    # Default port is 8080
+    enabled: True                           # [Optional] Default is False
+    # port: 8080                            # [Optional] Default is 8080
 
   # Output data to InfluxDB
   - name: influxdb
-    enabled: False
+    enabled: False                          # [Optional] Default is False
-    token: "xxx"                    # [Required] API Token
+    token: "xxx"                            # [Required] API Token
-    url: "http://localhost:8086"    # [Optional] Default URL is http://localhost:8086
+    url: "http://localhost:8086"            # [Optional] Default URL is http://localhost:8086
-    org: "Default"                  # [Required] InfluxDB Organization
+    org: "Default"                          # [Required] InfluxDB Organization
-    bucket: "SunGather"             # [Required] InfluxDB Bucket
+    bucket: "SunGather"                     # [Required] InfluxDB Bucket
-    measurements:
+    measurements:                           # [Required] Registers to publish to bucket
       - point: "power"
         register: daily_power_yields
       - point: "power"
@@ -51,88 +53,104 @@ exports:
       - point: "temp"
         register: internal_temperature
 
-
   # Publish Registers to MQTT / Home Assistant
   - name: mqtt
-    host: 192.168.1.200
+    enabled: False                          # [Optional] Default is False
-    # port: 1883
+    host: 192.168.1.200                     # [Required] IP or Hostname of MQTT Server 
-    topic: "tele/inverter_{model}/SENSOR" # Variable {model} will be replaced with model number
+    # port: 1883                            # [Optional] Default 1883
-    # username: 
+    # topic: "tele/inverter_{model}/SENSOR" # [Optional] Default: "tele/inverter_{model}/SENSOR", Variable {model} will be replaced with model number of inverter
-    # password:
+    # username:                             # [Optional] Username is MQTT server requires it
-    ha_discovery: True                    # Home Assistant Discovery, False by default
+    # password:                             # [Optional] Password is MQTT server requires it
-    ha_topics:                            # Name in HA, from Register
+    ha_discovery: True                      # [Optional] Default False, Home Assistant Discovery
+    ha_topics:                              # [Optional] / [Required] ha_discovery: True, Topics to enable discovery for HA
       - name: "Daily Generation"
+        sensor_type: sensor
         register: daily_power_yields
         unit: kWh
         dev_class: energy
         state_class: total_increasing
       - name: "Active Power"
+        sensor_type: sensor
         register: total_active_power
         unit: W
         dev_class: power
         state_class: measurement
       - name: "Load Power"
+        sensor_type: sensor
         register: load_power
         unit: W
         dev_class: power
         state_class: measurement
       - name: "Meter Power"
+        sensor_type: sensor
         register: meter_power
         unit: W
         dev_class: power
         state_class: measurement
       - name: "Export to Grid"
+        sensor_type: sensor
         register: export_to_grid
         unit: W
         dev_class: power
         state_class: measurement
       - name: "Import from Grid"
+        sensor_type: sensor
         register: import_from_grid
         unit: W
         dev_class: power
         state_class: measurement
       - name: "Temperature"
+        sensor_type: sensor
         register: internal_temperature
         unit: °C
         dev_class: temperature
         state_class: measurement
+      - name: "Power State"
+        sensor_type: binary_sensor
+        register: start_stop
+        dev_class: running
+        payload_on: "Start"
+        payload_off: "Stop"
 
-  - name: pvoutput      # Publish Registers to PVOutput
+  # Publish Registers to PVOutput
-    api: "xxxxx"
+  - name: pvoutput      
-    sid: "xxxxx"
+    enabled: False                          # [Optional] Default is False
-    # 60 for regular accounts, 300 for donation accounts
+    api: "xxxxx"                            # [Required] API Key, Settings > API Key
-    rate_limit: 60
+    sid: "xxxxx"                            # [Optional] System ID, Settings > Registered Systems > System ID
-    enabled: True
+    rate_limit: 60                          # [Optional] Default 60, 60 for regular accounts, 300 for donation accounts
-    parameters:
+    status_interval: 5                      # [Optional] Default 5, In minutes, options are 5 ,10,15 minute intervals: https://pvoutput.org/help/live_data.html
-      - name: v1    # Energy Generation
+    batch_points: 1                         # [Optional] Default 1, how many data points to batch upload, 
-        register: daily_power_yields    # Solar Generated Today (Energy)
+                                            # Time between uploads will be status_interval * batch_points. e.g. status_invterval of 5min, and batch_points of 12 will upload to PVOutput Hourly (5 * 12 = 60 mins)
+    parameters:                             # [Required] v1 & v3 or v2 & v4 minimum. See: https://pvoutput.org/help/api_specification.html#power-and-energy-calculation
+      - name: v1                            # Energy Generation
+        register: daily_power_yields        # Solar Generated Today (Energy)
         multiple: 1000
-      - name: v2    # Power Generation
+      - name: v2                            # Power Generation
-        register: total_active_power    # Current Generation (Power)
+        register: total_active_power        # Current Generation (Power)
-#     - name: v3    # Energy Consumption
+#     - name: v3                            # Energy Consumption
 #       register: 
-      - name: v4    # Power Consumption
+      - name: v4                            # Power Consumption
-        register: load_power            # Current Home usage (Power)
+        register: load_power                # Current Home usage (Power)
-      - name: v5    # Temperature
+      - name: v5                            # Temperature
-        register: internal_temperature  # Inverter internal temperature
+        register: internal_temperature      # Inverter internal temperature
-#     - name: v6    # Voltage
+#     - name: v6                            # Voltage
 #       register: total_active_power
-      - name: c1    # Cumulative Flag, 
+      - name: c1                            # Cumulative Flag, 
-        value: 1    # If using v2/v4 set to 1
+        value: 1                            # If using v2/v4 set to 1
-#      - name: "n"   # Net Flag
+#      - name: "n"                          # Net Flag
 #        value: 1
-#     - name: "v7"  # Extended Value v7 - Donation Only
+#     - name: "v7"                          # Extended Value v7 - Donation Only
 #       register:
-#     - name: "v8"  # Extended Value v8 - Donation Only
+#     - name: "v8"                          # Extended Value v8 - Donation Only
 #       register:
-#     - name: "v9"  # Extended Value v9 - Donation Only
+#     - name: "v9"                          # Extended Value v9 - Donation Only
 #       register:
-#     - name: "v10" # Extended Value v10 - Donation Only
+#     - name: "v10"                         # Extended Value v10 - Donation Only
 #       register:
-#     - name: "v11" # Extended Value v11 - Donation Only
+#     - name: "v11"                         # Extended Value v11 - Donation Only
 #       register:
-#     - name: "v12" # Extended Value v12 - Donation Only
+#     - name: "v12"                         # Extended Value v12 - Donation Only
 #       register:
-#     - name: "m1"  # Text Message 1 - Donation Only
+#     - name: "m1"                          # Text Message 1 - Donation Only
 #       register:
   

SunGather/exports/console.py

@@ -4,9 +4,12 @@ class export_console(object):
     def __init__(self):
         return
 
-    # Configure MQTT
+    # Configure Console
     def configure(self, config, config_inverter):
-        logging.info("Configured Console Logging")
+        logging.info("Console: Configured")
+        print("{:<20} {:<25}".format('Config','Value'))
+        for setting in config_inverter:
+            print("{:<40} {:<25}".format(setting,str(config_inverter.get(setting))))
 
     def publish(self, inverter):
 

SunGather/exports/influxdb.py

@@ -1,4 +1,3 @@
-from typing import Sequence
 import influxdb_client
 import logging
 from influxdb_client.client.write_api import SYNCHRONOUS
@@ -27,16 +26,16 @@ class export_influxdb(object):
             self.measurements.append(measurement)
 
         self.write_api = self.client.write_api(write_options=SYNCHRONOUS)
-        logging.info(f"Configured InfluxDB Client: {self.client.url}")
+        logging.info(f"InfluxDB: Configured: {self.client.url}")
 
     def publish(self, inverter):
         sequence = []
         for measurement in self.measurements:
             sequence.append(f"{measurement.get('point')},inverter={inverter.get('device_type_code', 'unknown').replace('.','').replace('-','')} {measurement.get('register')}={inverter.get(measurement.get('register'))}")
-        logging.info(f'InfluxDB Write: {sequence}')
+        logging.debug(f'InfluxDB: Sequence; {sequence}')
         try:
             self.write_api.write(self.bucket, self.client.org, sequence)
         except Exception as err:
-            logging.error(err)
+            logging.error("InfluxDB: " + str(err))
 
-        logging.info("Published to InfluxDB")
+        logging.info("InfluxDB: Published")

SunGather/exports/mqtt.py

@@ -6,9 +6,12 @@ class export_mqtt(object):
     def __init__(self):
         self.mqtt_client = None
         self.sensor_topic = None
-        self.ha_discovery = False
+        self.homeassistant = False
-        self.ha_topics = []
+        self.ha_discovery = True
-        self.update_model = True
+        self.ha_sensors = []
+        self.model = None
+        self.model_clean = None
+        self.inverter_ip = None
 
     # Configure MQTT
     def configure(self, config, config_inverter):
@@ -22,43 +25,66 @@ class export_mqtt(object):
         
         self.mqtt_client.connect(config.get('host'), port=config.get('port', 1883))
 
-        self.sensor_topic = config.get('topic', 'tele/inverter_{model}/SENSOR')
+        self.inverter_ip = config_inverter.get('host')
-        self.ha_discovery = config.get('ha_discovery', False)
 
-        if self.ha_discovery:
+        self.sensor_topic = config.get('topic', 'inverter/{model}/registers')
-            for ha_topic in config.get('ha_topics'):
+        self.homeassistant = config.get('homeassistant', False)
-                self.ha_topics.append(ha_topic)
 
-        logging.info(f"Configured MQTT Client: {config.get('host')}:{config.get('port', 1883)}")
+        if self.homeassistant:
+            for ha_sensor in config.get('ha_sensors'):
+                self.ha_sensors.append(ha_sensor)
+
+        logging.info(f"MQTT: Configured {config.get('host')}:{config.get('port', 1883)}")
 
     def publish(self, inverter):
         global mqtt_client
 
-        if (self.update_model):
-            self.sensor_topic = self.sensor_topic.replace('{model}', inverter.get('device_type_code', 'unknown').replace('.','').replace('-',''))
-            self.update_model = False
-
-        if self.ha_discovery:
-            self.mqtt_client.reconnect()
-            logging.info("Publishing Home Assistant Discovery messages")
-            discovery_topic = 'homeassistant/sensor/inverter/{}/config'
-            discovery_payload = '{{"name":"Inverter {}", "uniq_id":"{}", "stat_t":"{}", "json_attr_t":"{}", "unit_of_meas":"{}", "dev_cla":"{}", "state_class":"{}", "val_tpl":"{{{{ value_json.{} }}}}", "ic":"mdi:solar-power", "device":{{ "name":"Solar Inverter", "mf":"Sungrow", "mdl":"{}", "connections":[["address", "' + self.mqtt_client._host + '" ]]}}}}'   
-
-            for ha_topic in self.ha_topics:
-                msg = discovery_payload.format( ha_topic.get('name'),  "inverter_" + ha_topic.get('name').lower().replace(' ','_'),    self.sensor_topic,   self.sensor_topic,  ha_topic.get('unit'),   ha_topic.get('dev_class'),  ha_topic.get('state_class'),    ha_topic.get('register'), inverter.get('device_type_code', 'unknown').replace('.',''))
-                result = self.mqtt_client.publish(discovery_topic.format(ha_topic.get('name').lower().replace(' ','_')), msg, retain=True)
-                result.wait_for_publish()
-            self.ha_discovery = False
-
         # After a while you'll need to reconnect, so just reconnect before each publish
         self.mqtt_client.reconnect()
+
+        if not self.model:
+            self.model = inverter.get('device_type_code', 'unknown')
+            self.model_clean = self.model.replace('.','').replace('-','')
+            self.sensor_topic = self.sensor_topic.replace('{model}', self.model_clean)
+
+            logging.debug(f'MQTT: Sensor Topic = {self.sensor_topic}')
+
+        logging.debug(f"MQTT: Publishing: {self.sensor_topic} : {json.dumps(inverter)}")
         result = self.mqtt_client.publish(self.sensor_topic, json.dumps(inverter).replace('"', '\"'))
         result.wait_for_publish()
 
+        if self.homeassistant:
+            if self.ha_discovery:
+                for ha_sensor in self.ha_sensors:
+                    config_msg = {}
+                    if ha_sensor.get('name'):
+                        ha_topic = 'homeassistant/' + ha_sensor.get('sensor_type', 'sensor') + '/inverter/' + ha_sensor.get('name').lower().replace(' ','_') + '/config'
+                        config_msg['name'] = "Inverter " + ha_sensor.get('name')
+                        config_msg['unique_id'] = "inverter_" + ha_sensor.get('name').lower().replace(' ','_')
+                    config_msg['state_topic'] = self.sensor_topic
+                    config_msg['value_template'] = "{{ value_json." + ha_sensor.get('register') + " }}"
+                    if ha_sensor.get('unit'):
+                        config_msg['unit_of_measurement'] = ha_sensor.get('unit')
+                    if ha_sensor.get('dev_class'):
+                        config_msg['device_class'] = ha_sensor.get('dev_class')
+                    if ha_sensor.get('state_class'):
+                        config_msg['state_class'] = ha_sensor.get('state_class')
+                    if ha_sensor.get('payload_on'):
+                        config_msg['payload_on'] = ha_sensor.get('payload_on')
+                    if ha_sensor.get('payload_off'):
+                        config_msg['payload_off'] = ha_sensor.get('payload_off')
+                    config_msg['ic'] = "mdi:solar-power"
+                    config_msg['device'] = { "name":"Solar Inverter", "mf":"Sungrow", "mdl":self.model, "connections":[["address", self.inverter_ip ]]}
+
+                    result = self.mqtt_client.publish(ha_topic, json.dumps(config_msg), retain=True)
+                    result.wait_for_publish()
+                logging.info("MQTT: Published Home Assistant Discovery messages")
+                self.ha_discovery = False
+
         if result.rc != mqtt.MQTT_ERR_SUCCESS:
             # See https://github.com/eclipse/paho.mqtt.python/blob/master/src/paho/mqtt/client.py#L149 for error code mapping
-            logging.error(f"Failed to publish to MQTT with error code: {result.rc}")
+            logging.error(f"MQTT: Failed to publish with error code: {result.rc}")
         else:
-            logging.info("Published to MQTT")
+            logging.info("MQTT: Published")
 
         return result

SunGather/exports/pvoutput.py

@@ -7,8 +7,7 @@ class export_pvoutput(object):
     def __init__(self):
         self.api_key = None
         self.system_id = None
-        self.status_url = "https://pvoutput.org/service/r2/addstatus.jsp"
+        self.statusbatch_url = "https://pvoutput.org/service/r2/addbatchstatus.jsp"
-        self.metric_mappings = None
         self.rate_limit = None
         self.parameters = []
         self.latest_run = None
@@ -26,13 +25,22 @@ class export_pvoutput(object):
         self.api_key = config.get('api')
         self.system_id = config.get('sid')
         self.rate_limit = config.get('rate_limit', 60)
+        self.status_interval = config.get('status_interval',5)
+        self.batch_points = config.get('batch_points',1)
+        self.inverter_data = { "v1": 0, "v2": 0, "v3": 0, "v4": 0, "v5": 0, "v6": 0, "v7": 0, "v8": 0, "v9": 0, "v10": 0, "v11": 0, "v12": 0 }
+        self.inverter_data_count = 0
+        self.payload_data = None
+        self.batch_count = 0
+        self.payload_format = "{date},{time},{v1},{v2},{v3},{v4},{v5},{v6},{v7},{v8},{v9},{v10},{v11},{v12}"
 
         for parameter in config.get('parameters'):
             self.parameters.append(parameter)
 
-        logging.info("Configured PVOutput Client")
+        if not (self.status_interval == 5 or self.status_interval == 10 or self.status_interval == 15):
-
+            logging.warning("Status Invterval is invalid, valid options are 5, 10 and 15 minutes")
+            return False
 
+        logging.info("PVOutput: Configured")
 
     def publish(self, inverter):
         """
@@ -61,44 +69,65 @@ class export_pvoutput(object):
         v12         Extended Value v12  No          number      User Defined    Yes
         m1          Text Message 1      No          text        30 chars max    Yes
         """
-        at_least_one_of = set(["v1", "v2", "v3", "v4"])
 
         now = datetime.datetime.strptime(inverter.get('timestamp'), "%Y-%m-%d %H:%M:%S")
+        if not self.latest_run:     # Set last run to 1 min ago if never run, that way we don't miss any uploads
+            self.latest_run = now - datetime.timedelta(minutes=1)
 
-        if self.latest_run:
+        # Add new data to old data and increase count of data points
-            # Spread out our publishes over the hour based on the rate limit
-            time_diff = (now - self.latest_run).total_seconds()
-            if time_diff < (3600 / self.rate_limit):
-                return "skipped"
-
-        payload = {
-            "d": now.strftime("%Y%m%d"),
-            "t": now.strftime("%H:%M"),
-        }
-
-        value_present = False
         for parameter in self.parameters:
-            if parameter.get('name') == 'v1' or parameter.get('name') == 'v2' or parameter.get('name') == 'v3' or parameter.get('name') == 'v4':
+            if parameter.get('name')[0] == 'v':
-                value_present = True
+                if inverter.get(parameter.get('register')):
-            if parameter.get('register'):
+                    if parameter.get('multiple'):
-                if parameter.get('multiple'):
+                        self.inverter_data[parameter.get('name')] = round(self.inverter_data[parameter.get('name')] + (inverter.get(parameter.get('register')) * parameter.get('multiple')),3)
-                    payload.update({parameter.get('name'): str(inverter.get(parameter.get('register')) * parameter.get('multiple'))})
+                    else:
+                        self.inverter_data[parameter.get('name')] = round(self.inverter_data[parameter.get('name')] + inverter.get(parameter.get('register')),3)
                 else:
-                    payload.update({parameter.get('name'): str(inverter.get(parameter.get('register')))})
+                    logging.warning(f"PVOutput: {parameter.get('name')} configured to use {parameter.get('register')} but inverter is not returning this register")
-            elif parameter.get('value'):
+            elif parameter.get('name') == 'c1':
-                payload.update({parameter.get('name'): parameter.get('value')})
+                c1 = parameter.get('value')
+        self.inverter_data_count +=1
+        logging.info('PVOutput: Data logged')
+        logging.debug(f'PVOutput: Points: {self.batch_count} Data: {self.inverter_data}, c1={str(c1)}')
 
-        if not value_present:
+        # Process data points every status_interval
-            logging.error("PVOutput mapping failed, please review metric names and update")
+        if int(now.strftime("%M")) % self.status_interval == 0 and not now.strftime("%H:%M") == self.latest_run.strftime("%H:%M"):
-            return False
+            for v in self.inverter_data:
+                if not self.inverter_data[v] == 0:
+                    self.inverter_data[v] = round(self.inverter_data[v] / self.inverter_data_count, 2)
+            self.inverter_data_count = 0
 
-        logging.debug("PVOutput Request: " + self.status_url + ", " + str(self.headers) + " : " + str(payload))
+            payload = self.payload_format
-        response = requests.post(url=self.status_url, headers=self.headers, params=payload)
+            payload = payload.replace('{date}',now.strftime("%Y%m%d"))
+            payload = payload.replace('{time}',now.strftime("%H:%M"))
 
-        if response.status_code != requests.codes.ok:
+            for data in self.inverter_data:
-            raise RuntimeError(response.text)
+                payload = payload.replace('{' + data + '}',str(self.inverter_data.get(data)))
+
+            self.inverter_data = { "v1": 0, "v2": 0, "v3": 0, "v4": 0, "v5": 0, "v6": 0, "v7": 0, "v8": 0, "v9": 0, "v10": 0, "v11": 0, "v12": 0 }
+
+            if self.payload_data:
+                self.payload_data = self.payload_data + ";"
+            else: 
+                self.payload_data = ""
+            self.payload_data = self.payload_data + payload
+
+            self.batch_count +=1
+
+            if self.batch_count == self.batch_points:
+                payload = {'data': self.payload_data, 'c1': str(c1)}
+                logging.debug("PVOutput: Request; " + self.statusbatch_url + ", " + str(self.headers) + " : " + str(payload))
+                response = requests.post(url=self.statusbatch_url, headers=self.headers, params=payload)
+                self.batch_count = 0
+
+                if response.status_code != requests.codes.ok:
+                    raise RuntimeError(response.text)
+                else:
+                    self.payload_data = None
+                    logging.info("PVOutput: Data uploaded")
+            else:
+                logging.info("PVOutput: Data added to next batch upload")
 
-        logging.info("Published to PVOutput")
         self.latest_run = now
 
 

SunGather/exports/webserver.py

@@ -5,30 +5,30 @@ from version import __version__
 
 import logging
 
-class export_http(object):
+class export_webserver(object):
     html_body = "Pending Data Retrieval"
     def __init__(self):
         return
 
-    # Configure HTTP
+    # Configure Webserver
     def configure(self, config, config_inverter):
         self.webServer = HTTPServer(('', config.get('port',8080)), MyServer)
-        t = Thread(target=self.webServer.serve_forever)
+        self.t = Thread(target=self.webServer.serve_forever)
-        t.start()
+        self.t.start()
         self.config_inverter = config_inverter
-        logging.info("Configured Simple HTTP Server")
+        logging.info(f"Webserver: Configured")
 
     def publish(self, inverter):
         body = "<h3>Sungather v" + __version__ + "</h3></p><table><tr><th>Register</th><th>Value</th>"
         for register in inverter:
             body = body + f"<tr><td>{register}</td><td>{inverter.get(register)}</td></tr>"
-        export_http.html_body = body + f"</table><p>Total {len(inverter)} registers"
+        export_webserver.html_body = body + f"</table><p>Total {len(inverter)} registers"
 
         body = "</p></p><table><tr><th>Configuration</th><th>Value</th>"
         for config in self.config_inverter:
             body = body + f"<tr><td>{config}</td><td>{self.config_inverter.get(config)}</td></tr>"
-        export_http.html_body = export_http.html_body + body + f"</table></p>"
+        export_webserver.html_body = export_webserver.html_body + body + f"</table></p>"
-        logging.info("Updated Webserver Content")
+        logging.info("Webserver: Content Updated")
         return
 
 class MyServer(BaseHTTPRequestHandler):
@@ -38,7 +38,7 @@ class MyServer(BaseHTTPRequestHandler):
         self.end_headers()
         self.wfile.write(bytes("<html><head><title>SunGather</title><meta http-equiv='refresh' content='15'></head>", "utf-8"))
         self.wfile.write(bytes("<body>", "utf-8"))
-        self.wfile.write(bytes(export_http.html_body, "utf-8"))
+        self.wfile.write(bytes(export_webserver.html_body, "utf-8"))
         self.wfile.write(bytes("</table>", "utf-8"))
         self.wfile.write(bytes("</body></html>", "utf-8"))
     def log_message(self, format, *args):

SunGather/registers.yaml

@@ -1,6 +1,7 @@
 registers:
   - read:
     - name: "serial_number"
+      level: 3
       address: 4990
       datatype: "UTF-8"
     - name: "device_type_code"
@@ -380,10 +381,10 @@ registers:
       address: 5045
       datatype: "U16" 
       models: ["SG30KTL", "SG10KTL", "SG12KTL", "SG15KTL", "SG20KTL", "SG30KU", "SG36KTL", "SG36KU", "SG40KTL", "SG40KTL-M", "SG50KTL-M", "SG60KTL-M", "SG60KU", "SG30KTL-M", "SG30KTL-M-V31", "SG33KTL-M", "SG36KTL-M", "SG33K3J", "SG49K5J", "SG34KJ", "LP_P34KSG", "SG50KTL-M-20", "SG60KTL", "SG80KTL", "SG80KTL-20", "SG60KU-M", "SG5KTL-MT", "SG6KTL-MT", "SG8KTL-M", "SG10KTL-M", "SG10KTL-MT", "SG12KTL-M", "SG15KTL-M", "SG17KTL-M", "SG20KTL-M", "SG80KTL-M", "SG111HV", "SG125HV", "SG125HV-20", "SG30CX", "SG33CX", "SG36CX-US", "SG40CX", "SG50CX", "SG60CX-US", "SG110CX", "SG250HX", "SG250HX-US", "SG100CX", "SG100CX-JP", "SG250HX-IN", "SG25CX-SA", "SG75CX", "SG3.0RT", "SG4.0RT", "SG5.0RT", "SG6.0RT", "SG7.0RT", "SG8.0RT", "SG10RT", "SG12RT", "SG15RT", "SG17RT", "SG20RT","SG5K-D","SG8K-D"]
-    - name: "Reserved"    
+#   - name: "Reserved"    
-      address: 5046 - 5048
+#     address: 5046 - 5048
-      datatype: "U16"
+#     datatype: "U16"
-      models: ["SG30KTL", "SG10KTL", "SG12KTL", "SG15KTL", "SG20KTL", "SG30KU", "SG36KTL", "SG36KU", "SG40KTL", "SG40KTL-M", "SG50KTL-M", "SG60KTL-M", "SG60KU", "SG30KTL-M", "SG30KTL-M-V31", "SG33KTL-M", "SG36KTL-M", "SG33K3J", "SG49K5J", "SG34KJ", "LP_P34KSG", "SG50KTL-M-20", "SG60KTL", "SG80KTL", "SG80KTL-20", "SG60KU-M", "SG5KTL-MT", "SG6KTL-MT", "SG8KTL-M", "SG10KTL-M", "SG10KTL-MT", "SG12KTL-M", "SG15KTL-M", "SG17KTL-M", "SG20KTL-M", "SG80KTL-M", "SG111HV", "SG125HV", "SG125HV-20", "SG30CX", "SG33CX", "SG36CX-US", "SG40CX", "SG50CX", "SG60CX-US", "SG110CX", "SG250HX", "SG250HX-US", "SG100CX", "SG100CX-JP", "SG250HX-IN", "SG25CX-SA", "SG75CX", "SG3.0RT", "SG4.0RT", "SG5.0RT", "SG6.0RT", "SG7.0RT", "SG8.0RT", "SG10RT", "SG12RT", "SG15RT", "SG17RT", "SG20RT","SG5K-D","SG8K-D"]
+#     models: ["SG30KTL", "SG10KTL", "SG12KTL", "SG15KTL", "SG20KTL", "SG30KU", "SG36KTL", "SG36KU", "SG40KTL", "SG40KTL-M", "SG50KTL-M", "SG60KTL-M", "SG60KU", "SG30KTL-M", "SG30KTL-M-V31", "SG33KTL-M", "SG36KTL-M", "SG33K3J", "SG49K5J", "SG34KJ", "LP_P34KSG", "SG50KTL-M-20", "SG60KTL", "SG80KTL", "SG80KTL-20", "SG60KU-M", "SG5KTL-MT", "SG6KTL-MT", "SG8KTL-M", "SG10KTL-M", "SG10KTL-MT", "SG12KTL-M", "SG15KTL-M", "SG17KTL-M", "SG20KTL-M", "SG80KTL-M", "SG111HV", "SG125HV", "SG125HV-20", "SG30CX", "SG33CX", "SG36CX-US", "SG40CX", "SG50CX", "SG60CX-US", "SG110CX", "SG250HX", "SG250HX-US", "SG100CX", "SG100CX-JP", "SG250HX-IN", "SG25CX-SA", "SG75CX", "SG3.0RT", "SG4.0RT", "SG5.0RT", "SG6.0RT", "SG7.0RT", "SG8.0RT", "SG10RT", "SG12RT", "SG15RT", "SG17RT", "SG20RT","SG5K-D","SG8K-D"]
     - name: "nominal_reactive_power"
       level: 2
       address: 5049
@@ -1192,66 +1193,39 @@ registers:
 #     address: 5155-5199
 scan: # these have to be 1 less than the first register   
   - read:
-    - start: "5000"
+    - start: 5000
-      range: "100"
+      range: 100
-      level: 1
+    - start: 5100
-    - start: "5100"
+      range: 100
-      range: "100"
+    - start: 5200
-      level: 1
+      range: 100
-    - start: "5200"
+    - start: 6099
-      range: "100"
+      range: 100
-      level: 3
+    - start: 6199
-    - start: "6099"
+      range: 100
-      range: "100"
+    - start: 6299
-      level: 1
+      range: 100
-      hybrid: true
+    - start: 6399
-    - start: "6199"
+      range: 100
-      range: "100"
+    - start: 6499
-      level: 1
+      range: 100
-      hybrid: true
+    - start: 6599
-    - start: "6299"
+      range: 100
-      range: "100"
+    - start: 6699
-      level: 1
+      range: 100
-      hybrid: true
+    - start: 6799
-    - start: "6399"
+      range: 100
-      range: "100"
+    - start: 7012
-      level: 1
+      range: 25
-      hybrid: true
+    - start: 12999
-    - start: "6499"
+      range: 125
-      range: "100"
-      level: 1
-      hybrid: true
-    - start: "6599"
-      range: "100"
-      level: 1
-      hybrid: true
-    - start: "6699"
-      range: "100"
-      level: 1
-      hybrid: true
-    - start: "6799"
-      range: "100"
-      level: 1
-      hybrid: true
-    - start: "7012"
-      range: "25"
-      level: 3
-    - start: "12999"
-      range: "125"
-      level: 1
-      hybrid: true
   - hold:
-#    - start: "4999"   # Get current date / time only
+    - start: 4999
-#      range: "7"
+      range: 10
-    - start: "4999"
+    - start: 5009
-      range: "10"
+      range: 10
-      level: 0
+    - start: 5034
-    - start: "5009"
+      range: 10
-      range: "10"
-      level: 2
-    - start: "5034"
-      range: "10"
-      level: 2
 
 # full Models Supported:
 #["SG30KTL","SG10KTL","SG12KTL","SG15KTL","SG20KTL","SG30KU","SG36KTL","SG36KU","SG40KTL","SG40KTL-M","SG50KTL-M","SG60KTL-M","SG60KU","SG30KTL-M","SG30KTL-M-V31","SG33KTL-M","SG36KTL-M","SG33K3J","SG49K5J","SG34KJ","LP_P34KSG","SG50KTL-M-20","SG60KTL","SG80KTL","SG80KTL-20","SG60KU-M","SG5KTL-MT","SG6KTL-MT","SG8KTL-M","SG10KTL-M","SG10KTL-MT","SG12KTL-M","SG15KTL-M","SG17KTL-M","SG20KTL-M","SG80KTL-M","SG111HV","SG125HV","SG125HV-20","SG30CX","SG33CX","SG36CX-US","SG40CX","SG50CX","SG60CX-US","SG110CX","SG250HX","SG250HX-US","SG100CX","SG100CX-JP","SG250HX-IN","SG25CX-SA","SG75CX","SG3.0RT","SG4.0RT","SG5.0RT","SG6.0RT","SG7.0RT","SG8.0RT","SG10RT","SG12RT","SG15RT","SG17RT","SG20RT"]

SunGather/sungather.py

@@ -17,6 +17,7 @@ import os
 
 config = None
 registers = None
+register_ranges = None
 inverter = None
 client = None
 
@@ -54,31 +55,15 @@ def connect_inverter():
 def scrape_inverter():
     global inverter
 
-    """ Connect to the inverter and scrape the metrics """
+    scrape_start = datetime.now()
     client.connect()
 
-    for scan in registers['scan']:
+    for range in register_ranges:
-        if scan.get('read'):
+        logging.debug(f'Scraping: {range.get("type")}, {range.get("start")}:{range.get("range")}')
-            for subscan in registers['scan'][0]['read']:
+        if not load_registers(range.get('type'), int(range.get('start')), int(range.get('range'))):
-                if subscan.get('level',3) <= config['inverter'].get('level',1) or config['inverter'].get('level',1) == 3:
+            return False
-                    if not subscan.get('hybrid', False):
+
-                        logging.debug(f'Scanning: read, {subscan.get("start")}:{subscan.get("range")}')
+    client.close()
-                        if not load_registers("read", int(subscan.get('start')), int(subscan.get('range'))):
-                            return False
-                    elif subscan.get('hybrid', False) and config['inverter'].get('hybrid',False):
-                        logging.debug(f'Scanning: read, {subscan.get("start")}:{subscan.get("range")}')
-                        if not load_registers("read", int(subscan.get('start')), int(subscan.get('range'))):
-                            return False
-        if scan.get('hold'):
-            for subscan in registers['scan'][1]['hold']:
-                    if not subscan.get('hybrid', False):
-                        logging.debug(f'Scanning: hold, {subscan.get("start")}:{subscan.get("range")}')
-                        if not load_registers("hold", int(subscan.get('start')), int(subscan.get('range'))):
-                            return False
-                    elif subscan.get('hybrid', False) and config['inverter'].get('hybrid',False):
-                        logging.debug(f'Scanning: hold, {subscan.get("start")}:{subscan.get("range")}')
-                        if not load_registers("hold", int(subscan.get('start')), int(subscan.get('range'))):
-                            return False
 
     # Create a registers for Power imported and exported to/from Grid
     if config['inverter'].get('level',1) >= 1:
@@ -141,7 +126,9 @@ def scrape_inverter():
         except Exception:
             pass
 
-    client.close()
+    scrape_end = datetime.now()
+    logging.info(f'Inventer: Successfully scrapped in {(scrape_end - scrape_start).seconds}.{(scrape_end - scrape_start).microseconds} secs')
+
     return True
 
 def load_registers(register_type, start, count=100):
@@ -174,8 +161,8 @@ def load_registers(register_type, start, count=100):
     for num in range(0, count):
         run = int(start) + num + 1
 
-        for register in registers['registers'][0]['read']:
+        for register in registers:
-            if register_type == "read" and register['address'] == run:
+            if register_type == register.get('type') and register['address'] == run:
                 register_name = register['name']
 
                 register_value = None
@@ -208,66 +195,15 @@ def load_registers(register_type, start, count=100):
                 if register.get('multiple'):
                     register_value = round(register_value * register.get('multiple'),2)
 
-                # Set the final register name and value after checking model, any adjustments above included 
+                # Set the final register value with adjustments above included 
-                if register.get('level',3) <= config['inverter'].get('level',1) or config['inverter'].get('level',1) == 3:
+                inverter[register_name] = register_value
-                    if register.get('smart_meter') and config['inverter'].get('smart_meter'):
+                break
-                        inverter[register_name] = register_value
-                    elif register.get('models') and not config['inverter'].get('level',1) == 3:
-                        for supported_model in register.get('models'):
-                            if supported_model == inverter.get('device_type_code'):
-                                inverter[register_name] = register_value
-                    else:
-                        inverter[register_name] = register_value
-
-        for register in registers['registers'][1]['hold']:
-            if register_type == "hold" and register['address'] == run:
-                register_name = register['name']
-
-                register_value = None
-                # Return the 32bit value if needed
-                if register.get('datatype') == "U32" or register.get('datatype') == "S32":
-                    u32_value = rr.registers[num+1]
-                    if u32_value and register_value:
-                        register_value = (u32_value * 65535) + register_value
-
-                # We convert a system response to a human value 
-                if register.get('datarange'):
-                    for value in register.get('datarange'):
-                        if value['response'] == rr.registers[num]:
-                            register_value = value['value']
-                if not register_value:
-                    register_value = rr.registers[num]
-
-                # Adjust the value if needed
-                if register.get('indicator'):
-                    indicator_value = rr.registers[num+1]
-                    if indicator_value == 65535:
-                        register_value = -1 * (65535 - register_value)
-
-                # If xFF (U 65535 / S 32767) then change to 0, looks better when logging / graphing
-                if register.get('datatype') == 'S16' and (register_value == 32767 or register_value == 65535):
-                    register_value = 0
-                elif (register.get('datatype') == 'U16' or register.get('datatype') == 'S32' or register.get('datatype') == 'U32') and register_value == 65535:
-                    register_value = 0
-
-                if register.get('multiple'):
-                    register_value = round(register_value * register.get('multiple',0),2)
-
-                # Set the final register name and value after checking model, any adjustments above included 
-                if register.get('level',3) <= config['inverter'].get('level',1) or config['inverter'].get('level',1) == 3:
-                    if register.get('smart_meter') and config['inverter'].get('smart_meter'):
-                        inverter[register_name] = register_value
-                    elif register.get('models') and not config['inverter'].get('level',1) == 3:
-                        for supported_model in register.get('models'):
-                            if supported_model == inverter.get('device_type_code'):
-                                inverter[register_name] = register_value
-                    else:
-                        inverter[register_name] = register_value
     return True
 
 def main():
     global config
     global registers
+    global register_ranges
     global inverter
     global client
 
@@ -319,7 +255,7 @@ def main():
             sys.exit(1)   
 
     try:
-        registers = yaml.safe_load(open('registers.yaml'))
+        registers_raw = yaml.safe_load(open('registers.yaml'))
         logging.info(f"Loaded registers: {os.getcwd()}/registers.yaml")
     except Exception as err:
         logging.error(f"Failed: Loading registers: {os.getcwd()}/registers.yaml {err}")
@@ -334,18 +270,21 @@ def main():
     if config.get('exports'):
         for export in config.get('exports'):
             try:
-                if export.get('enabled', True):
+                if export.get('enabled', False):
                     export_load = importlib.import_module("exports." + export.get('name'))
-                    logging.info(f"Loaded Export: exports\{export.get('name')}")
+                    logging.info(f"Loading Export: exports\{export.get('name')}")
                     exports.append(getattr(export_load, "export_" + export.get('name'))())
                     exports[-1].configure(export, config['inverter'])
-                    logging.info(f"Configured export: {export.get('name')}")
             except Exception as err:
                 logging.error(f"Failed loading export: {err}" +
                             f"\n\t\t\t     Please make sure {export.get('name')}.py exists in the exports folder")
 
     client = connect_inverter()
     inverter = {}
+    registers = [[]]
+    registers.pop()
+    register_ranges = [[]]
+    register_ranges.pop()
 
     # Inverter Model Scanning
     model = 'unknown'
@@ -353,7 +292,14 @@ def main():
         model = config['inverter'].get('model')
         logging.info(f'Bypassing Model Detection, Using config: {model}')
     else:
+        # Load just the register to detect model, then we can load the rest of registers based on returned model
+        for register in registers_raw['registers'][0]['read']:
+            if register.get('name') == "device_type_code":
+                register['type'] = "read"
+                registers.append(register)
+                break
         if load_registers("read", 4999, 1):
+            registers.pop() # Remove the register, as we only needed it once
             if isinstance(inverter.get('device_type_code'),int):
                 logging.warning(f"Unknown Type Code Detected: {inverter.get('device_type_code')}")
             else:
@@ -362,8 +308,66 @@ def main():
         else:
             logging.info(f'Model detection failed, please set model in config.py')
 
+    # Load register list based om name and value after checking model
+    for register in registers_raw['registers'][0]['read']:
+        if register.get('level',3) <= config['inverter'].get('level',1) or config['inverter'].get('level',1) == 3:
+            register['type'] = "read"
+            register.pop('level')
+            if register.get('smart_meter') and config['inverter'].get('smart_meter'):
+                register.pop('models')
+                registers.append(register)
+            elif register.get('models') and not config['inverter'].get('level',1) == 3:
+                for supported_model in register.get('models'):
+                    if supported_model == inverter.get('device_type_code'):
+                        register.pop('models')
+                        registers.append(register)
+            else:
+                registers.append(register)
+
+    for register in registers_raw['registers'][1]['hold']:
+        if register.get('level',3) <= config['inverter'].get('level',1) or config['inverter'].get('level',1) == 3:
+            register['type'] = "hold"
+            register.pop('level')
+            if register.get('smart_meter') and config['inverter'].get('smart_meter'):
+                register.pop('models')
+                registers.append(register)
+            elif register.get('models') and not config['inverter'].get('level',1) == 3:
+                for supported_model in register.get('models'):
+                    if supported_model == inverter.get('device_type_code'):
+                        register.pop('models')
+                        registers.append(register)
+            else:
+                registers.append(register)
+
+
+    # Load register list based om name and value after checking model
+    for register_range in registers_raw['scan'][0]['read']:
+        register_range_used = False
+        register_range['type'] = "read"
+        for register in registers:
+            if register.get("type") == register_range.get("type"):
+                if register.get('address') >= register_range.get("start") and register.get('address') <= (register_range.get("start") + register_range.get("range")):
+                    register_range_used = True
+                    continue
+        if register_range_used:
+            register_ranges.append(register_range)
+
+
+    for register_range in registers_raw['scan'][1]['hold']:
+        register_range_used = False
+        register_range['type'] = "hold"
+        for register in registers:
+            if register.get("type") == register_range.get("type"):
+                if register.get('address') >= register_range.get("start") and register.get('address') <= (register_range.get("start") + register_range.get("range")):
+                    register_range_used = True
+                    continue
+        if register_range_used:
+            register_ranges.append(register_range)
+    
     # Core monitoring loop
     while True:
+
+        loop_start = datetime.now()
         # Clear previous inverter values, keep the model
         inverter = {}
         inverter['device_type_code'] = model
@@ -373,6 +377,7 @@ def main():
             time.sleep(3)       # Wait 3 secs for connection, resvoles issues with modbus not reconnecting on failure
 
         # Scrape the inverter
+        
         success = scrape_inverter()
 
         if(success):
@@ -382,13 +387,19 @@ def main():
         else:
             client.close()
             client = None
-            logging.warning(f"Data collection failed, skipped exporting data. Retying in {config['inverter'].get('scan_interval', 30)}")
+            logging.warning(f"Data collection failed, skipped exporting data. Retying in {config['inverter'].get('scan_interval', 30)} secs")
 
-        if not 'runonce' in locals():
+
-            # Sleep until the next scan
+        loop_end = datetime.now()
-            time.sleep(config['inverter'].get('scan_interval', 30))
+        logging.debug(f'Processing Time: {(loop_end - loop_start).seconds}.{(loop_end - loop_start).microseconds} secs')
-        else:
+
+        if 'runonce' in locals():
             sys.exit(0)
         
+        # Sleep until the next scan
+        time.sleep(config['inverter'].get('scan_interval', 30))
+
 if __name__== "__main__":
     main()
+
+sys.exit()

SunGather/version.py

@@ -1 +1 @@
-__version__ = '0.1.3'
+__version__ = '0.2.0'