From bb8b224c70bca6a9cd2178b022cd35cd4aaab2cc Mon Sep 17 00:00:00 2001 From: bohdan-s Date: Thu, 13 Jan 2022 17:02:19 +1100 Subject: [PATCH] v0.2.0, Improved register detection, improved performance, MQTT and PVOutput rewrites --- SunGather/config-example.yaml | 146 ++++++++------- SunGather/exports/console.py | 7 +- SunGather/exports/influxdb.py | 11 +- SunGather/exports/mqtt.py | 80 +++++--- SunGather/exports/pvoutput.py | 95 ++++++---- SunGather/exports/{http.py => webserver.py} | 18 +- SunGather/registers.yaml | 100 ++++------ SunGather/sungather.py | 193 +++++++++++--------- SunGather/version.py | 2 +- 9 files changed, 356 insertions(+), 296 deletions(-) rename SunGather/exports/{http.py => webserver.py} (74%) diff --git a/SunGather/config-example.yaml b/SunGather/config-example.yaml index d2c8814..eb999e3 100644 --- a/SunGather/config-example.yaml +++ b/SunGather/config-example.yaml @@ -1,41 +1,43 @@ inverter: - host: 192.168.1.100 # [Required] IP Address of the Inverter or Dongle - # port: 502 # [Optional] Default for modbus is 502, for http is 8082 - # slave: 0x01 # [Optional] Default is 0x01 - # timeout: 10 # [Optional] Default is 10 - # scan_interval: 30 # [Optional] Default is 30 - connection: sungrow # [Required] options: modbus, sungrow, http - # model: "SG7.0RT" # [Optional] This is autodetected on startup, only needed if detection issues or for testing - # See model list here: https://github.com/bohdan-s/SunGather#supported - # smart_meter: True # [Optional] Default is False, Set to true if inverter supports reading grind / house consumption - # use_local_time: False # [Optional] Default False, Uses Inventer time, if try uses PC time when updating timestamps (e.g. PVOutput) - # hybrid: False # [Optional] Default false, if you have a Hybrid (battery) set to True - # manual_load: False # [Optional] Manually calculate load total_active_power + meter_power if the inverter does not supply it - # logging: 30 # [Optional] 10 = Debug, 20 = Info, 30 = Warning (default), 40 = Error - # level: 1 # [Optional] Set the amount of information to gather - # 0 = Model and Solar Generation, - # 1 (default) = Useful data, all required for exports, - # 2 everything your Inverter supports, - # 3 Everything from every register + host: 192.168.1.100 # [Required] IP Address of the Inverter or Dongle + # port: 502 # [Optional] Default for modbus is 502, for http is 8082 + # slave: 0x01 # [Optional] Default is 0x01 + # timeout: 10 # [Optional] Default is 10 + # scan_interval: 30 # [Optional] Default is 30 + connection: sungrow # [Required] options: modbus, sungrow, http + # model: "SG7.0RT" # [Optional] This is autodetected on startup, only needed if detection issues or for testing + # See model list here: https://github.com/bohdan-s/SunGather#supported + # smart_meter: True # [Optional] Default is False, Set to true if inverter supports reading grind / house consumption + # use_local_time: False # [Optional] Default False, Uses Inventer time, if try uses PC time when updating timestamps (e.g. PVOutput) + # manual_load: False # [Optional] Manually calculate load total_active_power + meter_power if the inverter does not supply it + # logging: 30 # [Optional] 10 = Debug, 20 = Info, 30 = Warning (default), 40 = Error + # level: 1 # [Optional] Set the amount of information to gather + # 0 = Model and Solar Generation, + # 1 (default) = Useful data, all required for exports, + # 2 everything your Inverter supports, + # 3 Everything from every register +# If you do not want to use a export, you can either remove the whole configuration block +# or set enabled: False exports: # Print Registers to console, good for debugging / troubleshooting - - name: console - enabled: False + - name: console + enabled: False # [Optional] Default is False # Runs a simple Webserver showing Config and last read registers - - name: http - enabled: True # Access at http://localhost:8080 or http://[serverip]:8080 - # port: 8080 # Default port is 8080 + # Access at http://localhost:8080 or http://[serverip]:8080 + - name: webserver + enabled: True # [Optional] Default is False + # port: 8080 # [Optional] Default is 8080 # Output data to InfluxDB - name: influxdb - enabled: False - token: "xxx" # [Required] API Token - url: "http://localhost:8086" # [Optional] Default URL is http://localhost:8086 - org: "Default" # [Required] InfluxDB Organization - bucket: "SunGather" # [Required] InfluxDB Bucket - measurements: + enabled: False # [Optional] Default is False + token: "xxx" # [Required] API Token + url: "http://localhost:8086" # [Optional] Default URL is http://localhost:8086 + org: "Default" # [Required] InfluxDB Organization + bucket: "SunGather" # [Required] InfluxDB Bucket + measurements: # [Required] Registers to publish to bucket - point: "power" register: daily_power_yields - point: "power" @@ -51,88 +53,104 @@ exports: - point: "temp" register: internal_temperature - # Publish Registers to MQTT / Home Assistant - name: mqtt - host: 192.168.1.200 - # port: 1883 - topic: "tele/inverter_{model}/SENSOR" # Variable {model} will be replaced with model number - # username: - # password: - ha_discovery: True # Home Assistant Discovery, False by default - ha_topics: # Name in HA, from Register + enabled: False # [Optional] Default is False + host: 192.168.1.200 # [Required] IP or Hostname of MQTT Server + # port: 1883 # [Optional] Default 1883 + # topic: "tele/inverter_{model}/SENSOR" # [Optional] Default: "tele/inverter_{model}/SENSOR", Variable {model} will be replaced with model number of inverter + # username: # [Optional] Username is MQTT server requires it + # password: # [Optional] Password is MQTT server requires it + ha_discovery: True # [Optional] Default False, Home Assistant Discovery + ha_topics: # [Optional] / [Required] ha_discovery: True, Topics to enable discovery for HA - name: "Daily Generation" + sensor_type: sensor register: daily_power_yields unit: kWh dev_class: energy state_class: total_increasing - name: "Active Power" + sensor_type: sensor register: total_active_power unit: W dev_class: power state_class: measurement - name: "Load Power" + sensor_type: sensor register: load_power unit: W dev_class: power state_class: measurement - name: "Meter Power" + sensor_type: sensor register: meter_power unit: W dev_class: power state_class: measurement - name: "Export to Grid" + sensor_type: sensor register: export_to_grid unit: W dev_class: power state_class: measurement - name: "Import from Grid" + sensor_type: sensor register: import_from_grid unit: W dev_class: power state_class: measurement - name: "Temperature" + sensor_type: sensor register: internal_temperature unit: °C dev_class: temperature state_class: measurement + - name: "Power State" + sensor_type: binary_sensor + register: start_stop + dev_class: running + payload_on: "Start" + payload_off: "Stop" - - name: pvoutput # Publish Registers to PVOutput - api: "xxxxx" - sid: "xxxxx" - # 60 for regular accounts, 300 for donation accounts - rate_limit: 60 - enabled: True - parameters: - - name: v1 # Energy Generation - register: daily_power_yields # Solar Generated Today (Energy) + # Publish Registers to PVOutput + - name: pvoutput + enabled: False # [Optional] Default is False + api: "xxxxx" # [Required] API Key, Settings > API Key + sid: "xxxxx" # [Optional] System ID, Settings > Registered Systems > System ID + rate_limit: 60 # [Optional] Default 60, 60 for regular accounts, 300 for donation accounts + status_interval: 5 # [Optional] Default 5, In minutes, options are 5 ,10,15 minute intervals: https://pvoutput.org/help/live_data.html + batch_points: 1 # [Optional] Default 1, how many data points to batch upload, + # Time between uploads will be status_interval * batch_points. e.g. status_invterval of 5min, and batch_points of 12 will upload to PVOutput Hourly (5 * 12 = 60 mins) + parameters: # [Required] v1 & v3 or v2 & v4 minimum. See: https://pvoutput.org/help/api_specification.html#power-and-energy-calculation + - name: v1 # Energy Generation + register: daily_power_yields # Solar Generated Today (Energy) multiple: 1000 - - name: v2 # Power Generation - register: total_active_power # Current Generation (Power) -# - name: v3 # Energy Consumption + - name: v2 # Power Generation + register: total_active_power # Current Generation (Power) +# - name: v3 # Energy Consumption # register: - - name: v4 # Power Consumption - register: load_power # Current Home usage (Power) - - name: v5 # Temperature - register: internal_temperature # Inverter internal temperature -# - name: v6 # Voltage + - name: v4 # Power Consumption + register: load_power # Current Home usage (Power) + - name: v5 # Temperature + register: internal_temperature # Inverter internal temperature +# - name: v6 # Voltage # register: total_active_power - - name: c1 # Cumulative Flag, - value: 1 # If using v2/v4 set to 1 -# - name: "n" # Net Flag + - name: c1 # Cumulative Flag, + value: 1 # If using v2/v4 set to 1 +# - name: "n" # Net Flag # value: 1 -# - name: "v7" # Extended Value v7 - Donation Only +# - name: "v7" # Extended Value v7 - Donation Only # register: -# - name: "v8" # Extended Value v8 - Donation Only +# - name: "v8" # Extended Value v8 - Donation Only # register: -# - name: "v9" # Extended Value v9 - Donation Only +# - name: "v9" # Extended Value v9 - Donation Only # register: -# - name: "v10" # Extended Value v10 - Donation Only +# - name: "v10" # Extended Value v10 - Donation Only # register: -# - name: "v11" # Extended Value v11 - Donation Only +# - name: "v11" # Extended Value v11 - Donation Only # register: -# - name: "v12" # Extended Value v12 - Donation Only +# - name: "v12" # Extended Value v12 - Donation Only # register: -# - name: "m1" # Text Message 1 - Donation Only +# - name: "m1" # Text Message 1 - Donation Only # register: \ No newline at end of file diff --git a/SunGather/exports/console.py b/SunGather/exports/console.py index c18f9d7..f396dc5 100644 --- a/SunGather/exports/console.py +++ b/SunGather/exports/console.py @@ -4,9 +4,12 @@ class export_console(object): def __init__(self): return - # Configure MQTT + # Configure Console def configure(self, config, config_inverter): - logging.info("Configured Console Logging") + logging.info("Console: Configured") + print("{:<20} {:<25}".format('Config','Value')) + for setting in config_inverter: + print("{:<40} {:<25}".format(setting,str(config_inverter.get(setting)))) def publish(self, inverter): diff --git a/SunGather/exports/influxdb.py b/SunGather/exports/influxdb.py index 9bbecd0..cf5a9a2 100644 --- a/SunGather/exports/influxdb.py +++ b/SunGather/exports/influxdb.py @@ -1,4 +1,3 @@ -from typing import Sequence import influxdb_client import logging from influxdb_client.client.write_api import SYNCHRONOUS @@ -12,7 +11,7 @@ class export_influxdb(object): # Configure InfluxDB def configure(self, config, config_inverter): - + if not config.get('token') and config.get('org') and config.get('bucket') and config.get('measurements'): logging.warning(f"InfluxDB: Please check configuration") return False @@ -27,16 +26,16 @@ class export_influxdb(object): self.measurements.append(measurement) self.write_api = self.client.write_api(write_options=SYNCHRONOUS) - logging.info(f"Configured InfluxDB Client: {self.client.url}") + logging.info(f"InfluxDB: Configured: {self.client.url}") def publish(self, inverter): sequence = [] for measurement in self.measurements: sequence.append(f"{measurement.get('point')},inverter={inverter.get('device_type_code', 'unknown').replace('.','').replace('-','')} {measurement.get('register')}={inverter.get(measurement.get('register'))}") - logging.info(f'InfluxDB Write: {sequence}') + logging.debug(f'InfluxDB: Sequence; {sequence}') try: self.write_api.write(self.bucket, self.client.org, sequence) except Exception as err: - logging.error(err) + logging.error("InfluxDB: " + str(err)) - logging.info("Published to InfluxDB") \ No newline at end of file + logging.info("InfluxDB: Published") \ No newline at end of file diff --git a/SunGather/exports/mqtt.py b/SunGather/exports/mqtt.py index e66bdba..e323a7c 100644 --- a/SunGather/exports/mqtt.py +++ b/SunGather/exports/mqtt.py @@ -6,9 +6,12 @@ class export_mqtt(object): def __init__(self): self.mqtt_client = None self.sensor_topic = None - self.ha_discovery = False - self.ha_topics = [] - self.update_model = True + self.homeassistant = False + self.ha_discovery = True + self.ha_sensors = [] + self.model = None + self.model_clean = None + self.inverter_ip = None # Configure MQTT def configure(self, config, config_inverter): @@ -22,43 +25,66 @@ class export_mqtt(object): self.mqtt_client.connect(config.get('host'), port=config.get('port', 1883)) - self.sensor_topic = config.get('topic', 'tele/inverter_{model}/SENSOR') - self.ha_discovery = config.get('ha_discovery', False) + self.inverter_ip = config_inverter.get('host') - if self.ha_discovery: - for ha_topic in config.get('ha_topics'): - self.ha_topics.append(ha_topic) + self.sensor_topic = config.get('topic', 'inverter/{model}/registers') + self.homeassistant = config.get('homeassistant', False) - logging.info(f"Configured MQTT Client: {config.get('host')}:{config.get('port', 1883)}") + if self.homeassistant: + for ha_sensor in config.get('ha_sensors'): + self.ha_sensors.append(ha_sensor) + + logging.info(f"MQTT: Configured {config.get('host')}:{config.get('port', 1883)}") def publish(self, inverter): global mqtt_client - if (self.update_model): - self.sensor_topic = self.sensor_topic.replace('{model}', inverter.get('device_type_code', 'unknown').replace('.','').replace('-','')) - self.update_model = False - - if self.ha_discovery: - self.mqtt_client.reconnect() - logging.info("Publishing Home Assistant Discovery messages") - discovery_topic = 'homeassistant/sensor/inverter/{}/config' - discovery_payload = '{{"name":"Inverter {}", "uniq_id":"{}", "stat_t":"{}", "json_attr_t":"{}", "unit_of_meas":"{}", "dev_cla":"{}", "state_class":"{}", "val_tpl":"{{{{ value_json.{} }}}}", "ic":"mdi:solar-power", "device":{{ "name":"Solar Inverter", "mf":"Sungrow", "mdl":"{}", "connections":[["address", "' + self.mqtt_client._host + '" ]]}}}}' - - for ha_topic in self.ha_topics: - msg = discovery_payload.format( ha_topic.get('name'), "inverter_" + ha_topic.get('name').lower().replace(' ','_'), self.sensor_topic, self.sensor_topic, ha_topic.get('unit'), ha_topic.get('dev_class'), ha_topic.get('state_class'), ha_topic.get('register'), inverter.get('device_type_code', 'unknown').replace('.','')) - result = self.mqtt_client.publish(discovery_topic.format(ha_topic.get('name').lower().replace(' ','_')), msg, retain=True) - result.wait_for_publish() - self.ha_discovery = False - # After a while you'll need to reconnect, so just reconnect before each publish self.mqtt_client.reconnect() + + if not self.model: + self.model = inverter.get('device_type_code', 'unknown') + self.model_clean = self.model.replace('.','').replace('-','') + self.sensor_topic = self.sensor_topic.replace('{model}', self.model_clean) + + logging.debug(f'MQTT: Sensor Topic = {self.sensor_topic}') + + logging.debug(f"MQTT: Publishing: {self.sensor_topic} : {json.dumps(inverter)}") result = self.mqtt_client.publish(self.sensor_topic, json.dumps(inverter).replace('"', '\"')) result.wait_for_publish() + if self.homeassistant: + if self.ha_discovery: + for ha_sensor in self.ha_sensors: + config_msg = {} + if ha_sensor.get('name'): + ha_topic = 'homeassistant/' + ha_sensor.get('sensor_type', 'sensor') + '/inverter/' + ha_sensor.get('name').lower().replace(' ','_') + '/config' + config_msg['name'] = "Inverter " + ha_sensor.get('name') + config_msg['unique_id'] = "inverter_" + ha_sensor.get('name').lower().replace(' ','_') + config_msg['state_topic'] = self.sensor_topic + config_msg['value_template'] = "{{ value_json." + ha_sensor.get('register') + " }}" + if ha_sensor.get('unit'): + config_msg['unit_of_measurement'] = ha_sensor.get('unit') + if ha_sensor.get('dev_class'): + config_msg['device_class'] = ha_sensor.get('dev_class') + if ha_sensor.get('state_class'): + config_msg['state_class'] = ha_sensor.get('state_class') + if ha_sensor.get('payload_on'): + config_msg['payload_on'] = ha_sensor.get('payload_on') + if ha_sensor.get('payload_off'): + config_msg['payload_off'] = ha_sensor.get('payload_off') + config_msg['ic'] = "mdi:solar-power" + config_msg['device'] = { "name":"Solar Inverter", "mf":"Sungrow", "mdl":self.model, "connections":[["address", self.inverter_ip ]]} + + result = self.mqtt_client.publish(ha_topic, json.dumps(config_msg), retain=True) + result.wait_for_publish() + logging.info("MQTT: Published Home Assistant Discovery messages") + self.ha_discovery = False + if result.rc != mqtt.MQTT_ERR_SUCCESS: # See https://github.com/eclipse/paho.mqtt.python/blob/master/src/paho/mqtt/client.py#L149 for error code mapping - logging.error(f"Failed to publish to MQTT with error code: {result.rc}") + logging.error(f"MQTT: Failed to publish with error code: {result.rc}") else: - logging.info("Published to MQTT") + logging.info("MQTT: Published") return result \ No newline at end of file diff --git a/SunGather/exports/pvoutput.py b/SunGather/exports/pvoutput.py index 31faf16..6a03d00 100644 --- a/SunGather/exports/pvoutput.py +++ b/SunGather/exports/pvoutput.py @@ -7,8 +7,7 @@ class export_pvoutput(object): def __init__(self): self.api_key = None self.system_id = None - self.status_url = "https://pvoutput.org/service/r2/addstatus.jsp" - self.metric_mappings = None + self.statusbatch_url = "https://pvoutput.org/service/r2/addbatchstatus.jsp" self.rate_limit = None self.parameters = [] self.latest_run = None @@ -26,13 +25,22 @@ class export_pvoutput(object): self.api_key = config.get('api') self.system_id = config.get('sid') self.rate_limit = config.get('rate_limit', 60) + self.status_interval = config.get('status_interval',5) + self.batch_points = config.get('batch_points',1) + self.inverter_data = { "v1": 0, "v2": 0, "v3": 0, "v4": 0, "v5": 0, "v6": 0, "v7": 0, "v8": 0, "v9": 0, "v10": 0, "v11": 0, "v12": 0 } + self.inverter_data_count = 0 + self.payload_data = None + self.batch_count = 0 + self.payload_format = "{date},{time},{v1},{v2},{v3},{v4},{v5},{v6},{v7},{v8},{v9},{v10},{v11},{v12}" for parameter in config.get('parameters'): self.parameters.append(parameter) - logging.info("Configured PVOutput Client") - + if not (self.status_interval == 5 or self.status_interval == 10 or self.status_interval == 15): + logging.warning("Status Invterval is invalid, valid options are 5, 10 and 15 minutes") + return False + logging.info("PVOutput: Configured") def publish(self, inverter): """ @@ -61,44 +69,65 @@ class export_pvoutput(object): v12 Extended Value v12 No number User Defined Yes m1 Text Message 1 No text 30 chars max Yes """ - at_least_one_of = set(["v1", "v2", "v3", "v4"]) now = datetime.datetime.strptime(inverter.get('timestamp'), "%Y-%m-%d %H:%M:%S") + if not self.latest_run: # Set last run to 1 min ago if never run, that way we don't miss any uploads + self.latest_run = now - datetime.timedelta(minutes=1) - if self.latest_run: - # Spread out our publishes over the hour based on the rate limit - time_diff = (now - self.latest_run).total_seconds() - if time_diff < (3600 / self.rate_limit): - return "skipped" - - payload = { - "d": now.strftime("%Y%m%d"), - "t": now.strftime("%H:%M"), - } - - value_present = False + # Add new data to old data and increase count of data points for parameter in self.parameters: - if parameter.get('name') == 'v1' or parameter.get('name') == 'v2' or parameter.get('name') == 'v3' or parameter.get('name') == 'v4': - value_present = True - if parameter.get('register'): - if parameter.get('multiple'): - payload.update({parameter.get('name'): str(inverter.get(parameter.get('register')) * parameter.get('multiple'))}) + if parameter.get('name')[0] == 'v': + if inverter.get(parameter.get('register')): + if parameter.get('multiple'): + self.inverter_data[parameter.get('name')] = round(self.inverter_data[parameter.get('name')] + (inverter.get(parameter.get('register')) * parameter.get('multiple')),3) + else: + self.inverter_data[parameter.get('name')] = round(self.inverter_data[parameter.get('name')] + inverter.get(parameter.get('register')),3) else: - payload.update({parameter.get('name'): str(inverter.get(parameter.get('register')))}) - elif parameter.get('value'): - payload.update({parameter.get('name'): parameter.get('value')}) + logging.warning(f"PVOutput: {parameter.get('name')} configured to use {parameter.get('register')} but inverter is not returning this register") + elif parameter.get('name') == 'c1': + c1 = parameter.get('value') + self.inverter_data_count +=1 + logging.info('PVOutput: Data logged') + logging.debug(f'PVOutput: Points: {self.batch_count} Data: {self.inverter_data}, c1={str(c1)}') - if not value_present: - logging.error("PVOutput mapping failed, please review metric names and update") - return False + # Process data points every status_interval + if int(now.strftime("%M")) % self.status_interval == 0 and not now.strftime("%H:%M") == self.latest_run.strftime("%H:%M"): + for v in self.inverter_data: + if not self.inverter_data[v] == 0: + self.inverter_data[v] = round(self.inverter_data[v] / self.inverter_data_count, 2) + self.inverter_data_count = 0 - logging.debug("PVOutput Request: " + self.status_url + ", " + str(self.headers) + " : " + str(payload)) - response = requests.post(url=self.status_url, headers=self.headers, params=payload) + payload = self.payload_format + payload = payload.replace('{date}',now.strftime("%Y%m%d")) + payload = payload.replace('{time}',now.strftime("%H:%M")) - if response.status_code != requests.codes.ok: - raise RuntimeError(response.text) + for data in self.inverter_data: + payload = payload.replace('{' + data + '}',str(self.inverter_data.get(data))) + + self.inverter_data = { "v1": 0, "v2": 0, "v3": 0, "v4": 0, "v5": 0, "v6": 0, "v7": 0, "v8": 0, "v9": 0, "v10": 0, "v11": 0, "v12": 0 } + + if self.payload_data: + self.payload_data = self.payload_data + ";" + else: + self.payload_data = "" + self.payload_data = self.payload_data + payload + + self.batch_count +=1 + + if self.batch_count == self.batch_points: + payload = {'data': self.payload_data, 'c1': str(c1)} + logging.debug("PVOutput: Request; " + self.statusbatch_url + ", " + str(self.headers) + " : " + str(payload)) + response = requests.post(url=self.statusbatch_url, headers=self.headers, params=payload) + self.batch_count = 0 + + if response.status_code != requests.codes.ok: + raise RuntimeError(response.text) + else: + self.payload_data = None + logging.info("PVOutput: Data uploaded") + else: + logging.info("PVOutput: Data added to next batch upload") - logging.info("Published to PVOutput") self.latest_run = now diff --git a/SunGather/exports/http.py b/SunGather/exports/webserver.py similarity index 74% rename from SunGather/exports/http.py rename to SunGather/exports/webserver.py index c770570..ae03aac 100644 --- a/SunGather/exports/http.py +++ b/SunGather/exports/webserver.py @@ -5,30 +5,30 @@ from version import __version__ import logging -class export_http(object): +class export_webserver(object): html_body = "Pending Data Retrieval" def __init__(self): return - # Configure HTTP + # Configure Webserver def configure(self, config, config_inverter): self.webServer = HTTPServer(('', config.get('port',8080)), MyServer) - t = Thread(target=self.webServer.serve_forever) - t.start() + self.t = Thread(target=self.webServer.serve_forever) + self.t.start() self.config_inverter = config_inverter - logging.info("Configured Simple HTTP Server") + logging.info(f"Webserver: Configured") def publish(self, inverter): body = "

Sungather v" + __version__ + "

" for register in inverter: body = body + f"" - export_http.html_body = body + f"
RegisterValue
{register}{inverter.get(register)}

Total {len(inverter)} registers" + export_webserver.html_body = body + f"

Total {len(inverter)} registers" body = "

" for config in self.config_inverter: body = body + f"" - export_http.html_body = export_http.html_body + body + f"
ConfigurationValue
{config}{self.config_inverter.get(config)}

" - logging.info("Updated Webserver Content") + export_webserver.html_body = export_webserver.html_body + body + f"

" + logging.info("Webserver: Content Updated") return class MyServer(BaseHTTPRequestHandler): @@ -38,7 +38,7 @@ class MyServer(BaseHTTPRequestHandler): self.end_headers() self.wfile.write(bytes("SunGather", "utf-8")) self.wfile.write(bytes("", "utf-8")) - self.wfile.write(bytes(export_http.html_body, "utf-8")) + self.wfile.write(bytes(export_webserver.html_body, "utf-8")) self.wfile.write(bytes("", "utf-8")) self.wfile.write(bytes("", "utf-8")) def log_message(self, format, *args): diff --git a/SunGather/registers.yaml b/SunGather/registers.yaml index 1b6517e..6e0b43c 100644 --- a/SunGather/registers.yaml +++ b/SunGather/registers.yaml @@ -1,6 +1,7 @@ registers: - read: - name: "serial_number" + level: 3 address: 4990 datatype: "UTF-8" - name: "device_type_code" @@ -380,10 +381,10 @@ registers: address: 5045 datatype: "U16" models: ["SG30KTL", "SG10KTL", "SG12KTL", "SG15KTL", "SG20KTL", "SG30KU", "SG36KTL", "SG36KU", "SG40KTL", "SG40KTL-M", "SG50KTL-M", "SG60KTL-M", "SG60KU", "SG30KTL-M", "SG30KTL-M-V31", "SG33KTL-M", "SG36KTL-M", "SG33K3J", "SG49K5J", "SG34KJ", "LP_P34KSG", "SG50KTL-M-20", "SG60KTL", "SG80KTL", "SG80KTL-20", "SG60KU-M", "SG5KTL-MT", "SG6KTL-MT", "SG8KTL-M", "SG10KTL-M", "SG10KTL-MT", "SG12KTL-M", "SG15KTL-M", "SG17KTL-M", "SG20KTL-M", "SG80KTL-M", "SG111HV", "SG125HV", "SG125HV-20", "SG30CX", "SG33CX", "SG36CX-US", "SG40CX", "SG50CX", "SG60CX-US", "SG110CX", "SG250HX", "SG250HX-US", "SG100CX", "SG100CX-JP", "SG250HX-IN", "SG25CX-SA", "SG75CX", "SG3.0RT", "SG4.0RT", "SG5.0RT", "SG6.0RT", "SG7.0RT", "SG8.0RT", "SG10RT", "SG12RT", "SG15RT", "SG17RT", "SG20RT","SG5K-D","SG8K-D"] - - name: "Reserved" - address: 5046 - 5048 - datatype: "U16" - models: ["SG30KTL", "SG10KTL", "SG12KTL", "SG15KTL", "SG20KTL", "SG30KU", "SG36KTL", "SG36KU", "SG40KTL", "SG40KTL-M", "SG50KTL-M", "SG60KTL-M", "SG60KU", "SG30KTL-M", "SG30KTL-M-V31", "SG33KTL-M", "SG36KTL-M", "SG33K3J", "SG49K5J", "SG34KJ", "LP_P34KSG", "SG50KTL-M-20", "SG60KTL", "SG80KTL", "SG80KTL-20", "SG60KU-M", "SG5KTL-MT", "SG6KTL-MT", "SG8KTL-M", "SG10KTL-M", "SG10KTL-MT", "SG12KTL-M", "SG15KTL-M", "SG17KTL-M", "SG20KTL-M", "SG80KTL-M", "SG111HV", "SG125HV", "SG125HV-20", "SG30CX", "SG33CX", "SG36CX-US", "SG40CX", "SG50CX", "SG60CX-US", "SG110CX", "SG250HX", "SG250HX-US", "SG100CX", "SG100CX-JP", "SG250HX-IN", "SG25CX-SA", "SG75CX", "SG3.0RT", "SG4.0RT", "SG5.0RT", "SG6.0RT", "SG7.0RT", "SG8.0RT", "SG10RT", "SG12RT", "SG15RT", "SG17RT", "SG20RT","SG5K-D","SG8K-D"] +# - name: "Reserved" +# address: 5046 - 5048 +# datatype: "U16" +# models: ["SG30KTL", "SG10KTL", "SG12KTL", "SG15KTL", "SG20KTL", "SG30KU", "SG36KTL", "SG36KU", "SG40KTL", "SG40KTL-M", "SG50KTL-M", "SG60KTL-M", "SG60KU", "SG30KTL-M", "SG30KTL-M-V31", "SG33KTL-M", "SG36KTL-M", "SG33K3J", "SG49K5J", "SG34KJ", "LP_P34KSG", "SG50KTL-M-20", "SG60KTL", "SG80KTL", "SG80KTL-20", "SG60KU-M", "SG5KTL-MT", "SG6KTL-MT", "SG8KTL-M", "SG10KTL-M", "SG10KTL-MT", "SG12KTL-M", "SG15KTL-M", "SG17KTL-M", "SG20KTL-M", "SG80KTL-M", "SG111HV", "SG125HV", "SG125HV-20", "SG30CX", "SG33CX", "SG36CX-US", "SG40CX", "SG50CX", "SG60CX-US", "SG110CX", "SG250HX", "SG250HX-US", "SG100CX", "SG100CX-JP", "SG250HX-IN", "SG25CX-SA", "SG75CX", "SG3.0RT", "SG4.0RT", "SG5.0RT", "SG6.0RT", "SG7.0RT", "SG8.0RT", "SG10RT", "SG12RT", "SG15RT", "SG17RT", "SG20RT","SG5K-D","SG8K-D"] - name: "nominal_reactive_power" level: 2 address: 5049 @@ -1192,66 +1193,39 @@ registers: # address: 5155-5199 scan: # these have to be 1 less than the first register - read: - - start: "5000" - range: "100" - level: 1 - - start: "5100" - range: "100" - level: 1 - - start: "5200" - range: "100" - level: 3 - - start: "6099" - range: "100" - level: 1 - hybrid: true - - start: "6199" - range: "100" - level: 1 - hybrid: true - - start: "6299" - range: "100" - level: 1 - hybrid: true - - start: "6399" - range: "100" - level: 1 - hybrid: true - - start: "6499" - range: "100" - level: 1 - hybrid: true - - start: "6599" - range: "100" - level: 1 - hybrid: true - - start: "6699" - range: "100" - level: 1 - hybrid: true - - start: "6799" - range: "100" - level: 1 - hybrid: true - - start: "7012" - range: "25" - level: 3 - - start: "12999" - range: "125" - level: 1 - hybrid: true + - start: 5000 + range: 100 + - start: 5100 + range: 100 + - start: 5200 + range: 100 + - start: 6099 + range: 100 + - start: 6199 + range: 100 + - start: 6299 + range: 100 + - start: 6399 + range: 100 + - start: 6499 + range: 100 + - start: 6599 + range: 100 + - start: 6699 + range: 100 + - start: 6799 + range: 100 + - start: 7012 + range: 25 + - start: 12999 + range: 125 - hold: -# - start: "4999" # Get current date / time only -# range: "7" - - start: "4999" - range: "10" - level: 0 - - start: "5009" - range: "10" - level: 2 - - start: "5034" - range: "10" - level: 2 + - start: 4999 + range: 10 + - start: 5009 + range: 10 + - start: 5034 + range: 10 # full Models Supported: #["SG30KTL","SG10KTL","SG12KTL","SG15KTL","SG20KTL","SG30KU","SG36KTL","SG36KU","SG40KTL","SG40KTL-M","SG50KTL-M","SG60KTL-M","SG60KU","SG30KTL-M","SG30KTL-M-V31","SG33KTL-M","SG36KTL-M","SG33K3J","SG49K5J","SG34KJ","LP_P34KSG","SG50KTL-M-20","SG60KTL","SG80KTL","SG80KTL-20","SG60KU-M","SG5KTL-MT","SG6KTL-MT","SG8KTL-M","SG10KTL-M","SG10KTL-MT","SG12KTL-M","SG15KTL-M","SG17KTL-M","SG20KTL-M","SG80KTL-M","SG111HV","SG125HV","SG125HV-20","SG30CX","SG33CX","SG36CX-US","SG40CX","SG50CX","SG60CX-US","SG110CX","SG250HX","SG250HX-US","SG100CX","SG100CX-JP","SG250HX-IN","SG25CX-SA","SG75CX","SG3.0RT","SG4.0RT","SG5.0RT","SG6.0RT","SG7.0RT","SG8.0RT","SG10RT","SG12RT","SG15RT","SG17RT","SG20RT"] \ No newline at end of file diff --git a/SunGather/sungather.py b/SunGather/sungather.py index f25eece..cb7befb 100644 --- a/SunGather/sungather.py +++ b/SunGather/sungather.py @@ -17,6 +17,7 @@ import os config = None registers = None +register_ranges = None inverter = None client = None @@ -54,31 +55,15 @@ def connect_inverter(): def scrape_inverter(): global inverter - """ Connect to the inverter and scrape the metrics """ + scrape_start = datetime.now() client.connect() - for scan in registers['scan']: - if scan.get('read'): - for subscan in registers['scan'][0]['read']: - if subscan.get('level',3) <= config['inverter'].get('level',1) or config['inverter'].get('level',1) == 3: - if not subscan.get('hybrid', False): - logging.debug(f'Scanning: read, {subscan.get("start")}:{subscan.get("range")}') - if not load_registers("read", int(subscan.get('start')), int(subscan.get('range'))): - return False - elif subscan.get('hybrid', False) and config['inverter'].get('hybrid',False): - logging.debug(f'Scanning: read, {subscan.get("start")}:{subscan.get("range")}') - if not load_registers("read", int(subscan.get('start')), int(subscan.get('range'))): - return False - if scan.get('hold'): - for subscan in registers['scan'][1]['hold']: - if not subscan.get('hybrid', False): - logging.debug(f'Scanning: hold, {subscan.get("start")}:{subscan.get("range")}') - if not load_registers("hold", int(subscan.get('start')), int(subscan.get('range'))): - return False - elif subscan.get('hybrid', False) and config['inverter'].get('hybrid',False): - logging.debug(f'Scanning: hold, {subscan.get("start")}:{subscan.get("range")}') - if not load_registers("hold", int(subscan.get('start')), int(subscan.get('range'))): - return False + for range in register_ranges: + logging.debug(f'Scraping: {range.get("type")}, {range.get("start")}:{range.get("range")}') + if not load_registers(range.get('type'), int(range.get('start')), int(range.get('range'))): + return False + + client.close() # Create a registers for Power imported and exported to/from Grid if config['inverter'].get('level',1) >= 1: @@ -141,7 +126,9 @@ def scrape_inverter(): except Exception: pass - client.close() + scrape_end = datetime.now() + logging.info(f'Inventer: Successfully scrapped in {(scrape_end - scrape_start).seconds}.{(scrape_end - scrape_start).microseconds} secs') + return True def load_registers(register_type, start, count=100): @@ -174,8 +161,8 @@ def load_registers(register_type, start, count=100): for num in range(0, count): run = int(start) + num + 1 - for register in registers['registers'][0]['read']: - if register_type == "read" and register['address'] == run: + for register in registers: + if register_type == register.get('type') and register['address'] == run: register_name = register['name'] register_value = None @@ -208,66 +195,15 @@ def load_registers(register_type, start, count=100): if register.get('multiple'): register_value = round(register_value * register.get('multiple'),2) - # Set the final register name and value after checking model, any adjustments above included - if register.get('level',3) <= config['inverter'].get('level',1) or config['inverter'].get('level',1) == 3: - if register.get('smart_meter') and config['inverter'].get('smart_meter'): - inverter[register_name] = register_value - elif register.get('models') and not config['inverter'].get('level',1) == 3: - for supported_model in register.get('models'): - if supported_model == inverter.get('device_type_code'): - inverter[register_name] = register_value - else: - inverter[register_name] = register_value - - for register in registers['registers'][1]['hold']: - if register_type == "hold" and register['address'] == run: - register_name = register['name'] - - register_value = None - # Return the 32bit value if needed - if register.get('datatype') == "U32" or register.get('datatype') == "S32": - u32_value = rr.registers[num+1] - if u32_value and register_value: - register_value = (u32_value * 65535) + register_value - - # We convert a system response to a human value - if register.get('datarange'): - for value in register.get('datarange'): - if value['response'] == rr.registers[num]: - register_value = value['value'] - if not register_value: - register_value = rr.registers[num] - - # Adjust the value if needed - if register.get('indicator'): - indicator_value = rr.registers[num+1] - if indicator_value == 65535: - register_value = -1 * (65535 - register_value) - - # If xFF (U 65535 / S 32767) then change to 0, looks better when logging / graphing - if register.get('datatype') == 'S16' and (register_value == 32767 or register_value == 65535): - register_value = 0 - elif (register.get('datatype') == 'U16' or register.get('datatype') == 'S32' or register.get('datatype') == 'U32') and register_value == 65535: - register_value = 0 - - if register.get('multiple'): - register_value = round(register_value * register.get('multiple',0),2) - - # Set the final register name and value after checking model, any adjustments above included - if register.get('level',3) <= config['inverter'].get('level',1) or config['inverter'].get('level',1) == 3: - if register.get('smart_meter') and config['inverter'].get('smart_meter'): - inverter[register_name] = register_value - elif register.get('models') and not config['inverter'].get('level',1) == 3: - for supported_model in register.get('models'): - if supported_model == inverter.get('device_type_code'): - inverter[register_name] = register_value - else: - inverter[register_name] = register_value + # Set the final register value with adjustments above included + inverter[register_name] = register_value + break return True def main(): global config global registers + global register_ranges global inverter global client @@ -319,7 +255,7 @@ def main(): sys.exit(1) try: - registers = yaml.safe_load(open('registers.yaml')) + registers_raw = yaml.safe_load(open('registers.yaml')) logging.info(f"Loaded registers: {os.getcwd()}/registers.yaml") except Exception as err: logging.error(f"Failed: Loading registers: {os.getcwd()}/registers.yaml {err}") @@ -334,18 +270,21 @@ def main(): if config.get('exports'): for export in config.get('exports'): try: - if export.get('enabled', True): + if export.get('enabled', False): export_load = importlib.import_module("exports." + export.get('name')) - logging.info(f"Loaded Export: exports\{export.get('name')}") + logging.info(f"Loading Export: exports\{export.get('name')}") exports.append(getattr(export_load, "export_" + export.get('name'))()) exports[-1].configure(export, config['inverter']) - logging.info(f"Configured export: {export.get('name')}") except Exception as err: logging.error(f"Failed loading export: {err}" + f"\n\t\t\t Please make sure {export.get('name')}.py exists in the exports folder") client = connect_inverter() inverter = {} + registers = [[]] + registers.pop() + register_ranges = [[]] + register_ranges.pop() # Inverter Model Scanning model = 'unknown' @@ -353,7 +292,14 @@ def main(): model = config['inverter'].get('model') logging.info(f'Bypassing Model Detection, Using config: {model}') else: + # Load just the register to detect model, then we can load the rest of registers based on returned model + for register in registers_raw['registers'][0]['read']: + if register.get('name') == "device_type_code": + register['type'] = "read" + registers.append(register) + break if load_registers("read", 4999, 1): + registers.pop() # Remove the register, as we only needed it once if isinstance(inverter.get('device_type_code'),int): logging.warning(f"Unknown Type Code Detected: {inverter.get('device_type_code')}") else: @@ -362,8 +308,66 @@ def main(): else: logging.info(f'Model detection failed, please set model in config.py') + # Load register list based om name and value after checking model + for register in registers_raw['registers'][0]['read']: + if register.get('level',3) <= config['inverter'].get('level',1) or config['inverter'].get('level',1) == 3: + register['type'] = "read" + register.pop('level') + if register.get('smart_meter') and config['inverter'].get('smart_meter'): + register.pop('models') + registers.append(register) + elif register.get('models') and not config['inverter'].get('level',1) == 3: + for supported_model in register.get('models'): + if supported_model == inverter.get('device_type_code'): + register.pop('models') + registers.append(register) + else: + registers.append(register) + + for register in registers_raw['registers'][1]['hold']: + if register.get('level',3) <= config['inverter'].get('level',1) or config['inverter'].get('level',1) == 3: + register['type'] = "hold" + register.pop('level') + if register.get('smart_meter') and config['inverter'].get('smart_meter'): + register.pop('models') + registers.append(register) + elif register.get('models') and not config['inverter'].get('level',1) == 3: + for supported_model in register.get('models'): + if supported_model == inverter.get('device_type_code'): + register.pop('models') + registers.append(register) + else: + registers.append(register) + + + # Load register list based om name and value after checking model + for register_range in registers_raw['scan'][0]['read']: + register_range_used = False + register_range['type'] = "read" + for register in registers: + if register.get("type") == register_range.get("type"): + if register.get('address') >= register_range.get("start") and register.get('address') <= (register_range.get("start") + register_range.get("range")): + register_range_used = True + continue + if register_range_used: + register_ranges.append(register_range) + + + for register_range in registers_raw['scan'][1]['hold']: + register_range_used = False + register_range['type'] = "hold" + for register in registers: + if register.get("type") == register_range.get("type"): + if register.get('address') >= register_range.get("start") and register.get('address') <= (register_range.get("start") + register_range.get("range")): + register_range_used = True + continue + if register_range_used: + register_ranges.append(register_range) + # Core monitoring loop while True: + + loop_start = datetime.now() # Clear previous inverter values, keep the model inverter = {} inverter['device_type_code'] = model @@ -373,6 +377,7 @@ def main(): time.sleep(3) # Wait 3 secs for connection, resvoles issues with modbus not reconnecting on failure # Scrape the inverter + success = scrape_inverter() if(success): @@ -382,13 +387,19 @@ def main(): else: client.close() client = None - logging.warning(f"Data collection failed, skipped exporting data. Retying in {config['inverter'].get('scan_interval', 30)}") + logging.warning(f"Data collection failed, skipped exporting data. Retying in {config['inverter'].get('scan_interval', 30)} secs") - if not 'runonce' in locals(): - # Sleep until the next scan - time.sleep(config['inverter'].get('scan_interval', 30)) - else: + + loop_end = datetime.now() + logging.debug(f'Processing Time: {(loop_end - loop_start).seconds}.{(loop_end - loop_start).microseconds} secs') + + if 'runonce' in locals(): sys.exit(0) + + # Sleep until the next scan + time.sleep(config['inverter'].get('scan_interval', 30)) if __name__== "__main__": - main() \ No newline at end of file + main() + +sys.exit() \ No newline at end of file diff --git a/SunGather/version.py b/SunGather/version.py index ef0b380..a9fdc5c 100644 --- a/SunGather/version.py +++ b/SunGather/version.py @@ -1 +1 @@ -__version__ = '0.1.3' \ No newline at end of file +__version__ = '0.2.0' \ No newline at end of file