Add TCP source for lightweight, extended data gathering

Certain Omnik inverters (supposedly with a specific serial number, see
sources below) reply with its statistics as binary data when a "magic"
packet is sent to them on port 8899 over TCP.  This data is more
lightweight than an HTML page or JavaScript source with values embedded,
and provides more statistics then either similar to what can be found on
the LCD display.

The data format and meaning has been extracted from various Python/C
sources (see links below), with some of my own "work" to simplify
checksum calculation and "the reverse hexadecimal serial number" is
nothing more than its little-endian byte representation.  Response data
is parsed at once with a `ctypes.BigEndianStructure` instead of
unpacking individual ranges of bytes through `struct.unpack`.  Only
minimal data massaging is needed to convert the struct to useful
information.

It provides the following extra fields:

- Inverter temperature;
- Voltage and current for the DC input strings (up to 3);
- Voltage, current, frequency and power for all AC outputs (also up to
  3);
- Total number of runtime hours (not sure if this is just the Omnik
  being on, or the time it delivers >0W of power back to the network).

No "device" (network module) data is found in any of the unparsed bits,
nor is `solar_rater_power` available.

Sources:

1. https://github.com/jbouwh/omnikdatalogger/blob/dev/apps/omnikdatalogger/omnik/InverterMsg.py
2. http://www.mb200d.nl/wordpress/2015/11/omniksol-4k-tl-wifi-kit/#more-590
3. https://github.com/Woutrrr/Omnik-Data-Logger
4. https://github.com/t3kpunk/Omniksol-PV-Logger

README.md

@@ -32,6 +32,7 @@ A python package with which you can read the data from your Omnik Inverter. Keep
 - Omnik1500TL
 - Omnik2000TL
 - Omnik2000TL2
+- Omnik3000TL
 - Omnik4000TL2
 - Ginlong stick (JSON)
 
@@ -68,7 +69,7 @@ if __name__ == "__main__":
     loop.run_until_complete(main())
 ```
 
-For the **source type** you can choose between: `javascript` (default), `json` and `html`.
+For the **source type** you can choose between: `javascript` (default), `json`, `html` and `tcp`.
 
 ## Data
 
@@ -85,6 +86,13 @@ You can read the following data with this package:
 - Day Energy Production (kWh)
 - Total Energy Production (kWh)
 
+On the `tcp` source type you can also find:
+
+- Inverter temperature;
+- Voltage and current for the DC input strings (up to 3)
+- Voltage, current, frequency and power for all AC outputs (also up to 3)
+- Total number of runtime hours.
+
 ### Device
 
 - Signal Quality (only with JS)

omnikinverter/exceptions.py

@@ -24,3 +24,11 @@ class OmnikInverterWrongValuesError(OmnikInverterError):
     shows the same value for both day and
     total production.
     """
+
+
+class OmnikInverterPacketInvalidError(OmnikInverterError):
+    """Omnik Inverter replied with invalid data.
+
+    This error appears on the "tcp" source when received
+    data cannot be validated.
+    """

omnikinverter/models.py

@@ -22,6 +22,20 @@ class Inverter:
     solar_energy_today: float | None
     solar_energy_total: float | None
 
+    # TCP only
+    inverter_active: bool | None = None
+    solar_hours_total: int | None = None
+
+    temperature: float | None = None
+
+    dc_input_voltage: list[float] | None = None
+    dc_input_current: list[float] | None = None
+
+    ac_output_voltage: list[float] | None = None
+    ac_output_current: list[float] | None = None
+    ac_output_frequency: list[float] | None = None
+    ac_output_power: list[float] | None = None
+
     @staticmethod
     def from_json(data: dict[str, Any]) -> Inverter:
         """Return Inverter object from the Omnik Inverter response.
@@ -156,6 +170,25 @@ def get_value(position):
             solar_energy_total=get_value(7),
         )
 
+    @staticmethod
+    def from_tcp(data: dict[str, Any]) -> Inverter:
+        """Return Inverter object from the Omnik Inverter response.
+
+        Args:
+            data: The binary data from the Omnik Inverter.
+
+        Returns:
+            An Inverter object.
+        """
+
+        return Inverter(
+            **data,
+            solar_rated_power=None,
+            solar_current_power=sum(
+                p for p in data["ac_output_power"] if p is not None
+            ),
+        )
+
 
 @dataclass
 class Device:

omnikinverter/omnikinverter.py

@@ -12,6 +12,7 @@
 from aiohttp.hdrs import METH_GET
 from yarl import URL
 
+from . import tcp
 from .exceptions import (
     OmnikInverterAuthError,
     OmnikInverterConnectionError,
@@ -30,6 +31,8 @@ class OmnikInverter:
     source_type: str = "javascript"
     request_timeout: int = 10
     session: ClientSession | None = None
+    serial_number: int | None = None  # Optional, only for TCP backend
+    tcp_port: int = 8899
 
     _close_session: bool = False
 
@@ -116,6 +119,38 @@ async def request(
 
         return raw_response.decode("ascii", "ignore")
 
+    async def tcp_request(self) -> dict[str, Any]:
+        """Perform a raw TCP request to the Omnik device.
+
+        Returns:
+            A Python dictionary (text) with the response from
+            the Omnik Inverter.
+
+        Raises:
+            OmnikInverterAuthError: Serial number is required to communicate
+                with the Omnik Inverter.
+            OmnikInverterConnectionError: An error occurred while communicating
+                with the Omnik Inverter.
+        """
+        if self.serial_number is None:
+            raise OmnikInverterAuthError("serial_number is missing from the request")
+
+        try:
+            reader, writer = await asyncio.open_connection(self.host, self.tcp_port)
+            writer.write(tcp.create_information_request(self.serial_number))
+            await writer.drain()
+
+            raw_msg = await reader.read(1024)
+        except Exception as exception:
+            raise OmnikInverterConnectionError(
+                "Failed to communicate with the Omnik Inverter device over TCP"
+            ) from exception
+        finally:
+            writer.close()
+            await writer.wait_closed()
+
+        return tcp.parse_information_reply(self.serial_number, raw_msg)
+
     async def inverter(self) -> Inverter:
         """Get values from your Omnik Inverter.
 
@@ -128,23 +163,28 @@ async def inverter(self) -> Inverter:
         if self.source_type == "html":
             data = await self.request("status.html")
             return Inverter.from_html(data)
-        data = await self.request("js/status.js")
-        return Inverter.from_js(data)
+        if self.source_type == "javascript":
+            data = await self.request("js/status.js")
+            return Inverter.from_js(data)
+        if self.source_type == "tcp":
+            data = await self.tcp_request()
+            return Inverter.from_tcp(data)
 
     async def device(self) -> Device:
         """Get values from the device.
 
         Returns:
-            A Device data object from the Omnik Inverter.
+            A Device data object from the Omnik Inverter. None on the "tcp" source_type.
         """
         if self.source_type == "json":
             data = await self.request("status.json", params={"CMD": "inv_query"})
             return Device.from_json(data)
         if self.source_type == "html":
             data = await self.request("status.html")
             return Device.from_html(data)
-        data = await self.request("js/status.js")
-        return Device.from_js(data)
+        if self.source_type == "javascript":
+            data = await self.request("js/status.js")
+            return Device.from_js(data)
 
     async def close(self) -> None:
         """Close open client session."""

omnikinverter/tcp.py

@@ -0,0 +1,201 @@
+"""Data model and conversions for tcp-based communication with the Omnik Inverter."""
+from ctypes import BigEndianStructure, c_char, c_ubyte, c_uint, c_ushort
+from typing import Any
+
+from .exceptions import OmnikInverterPacketInvalidError
+
+MESSAGE_END = 0x16
+MESSAGE_START = 0x68
+UINT16_MAX = 65535
+
+
+class _AcOutput(BigEndianStructure):
+    _fields_ = [
+        ("frequency", c_ushort),
+        ("power", c_ushort),
+    ]
+
+    def get_power(self):
+        """Retrieve AC power.
+
+        Returns:
+            The power field, or None if it is unset.
+        """
+        return None if self.power == UINT16_MAX else self.power
+
+    def get_frequency(self):
+        """Retrieve AC frequency.
+
+        Returns:
+            The frequency field in Hertz, or None if it is unset.
+        """
+        return None if self.frequency == UINT16_MAX else self.frequency * 0.01
+
+
+class _TcpData(BigEndianStructure):
+    _pack_ = 1
+    _fields_ = [
+        # Unlikely for all these bytes to represent the model,
+        # this mapping has to be built out over time.
+        ("model", c_char * 3),
+        # The s/n from the request, twice, in little-endian
+        ("double_serial_number", (c_ubyte * 4) * 2),
+        ("padding0", c_char * 3),
+        ("serial_number", c_char * 16),
+        ("temperature", c_ushort),
+        ("dc_input_voltage", c_ushort * 3),
+        ("dc_input_current", c_ushort * 3),
+        ("ac_output_current", c_ushort * 3),
+        ("ac_output_voltage", c_ushort * 3),
+        ("ac_output", _AcOutput * 3),
+        ("solar_energy_today", c_ushort),
+        ("solar_energy_total", c_uint),
+        ("solar_hours_total", c_uint),
+        ("inverter_active", c_ushort),
+        ("padding1", c_ubyte * 4),
+        ("unknown0", c_ushort),
+        ("padding2", c_ubyte * 10),
+        ("firmware", c_char * 16),
+        ("padding3", c_ubyte * 4),
+        ("firmware_slave", c_char * 16),
+        ("padding4", c_ubyte * 4),
+    ]
+
+
+def _pack_message(message: bytearray) -> bytearray:
+    checksum = sum(message) & 0xFF
+
+    message.insert(0, MESSAGE_START)
+    message.append(checksum)
+    message.append(MESSAGE_END)
+
+    return message
+
+
+def _unpack_message(message: bytearray) -> bytearray:
+    if message.pop(0) != MESSAGE_START:
+        raise OmnikInverterPacketInvalidError("Invalid start byte")
+    if message.pop() != MESSAGE_END:
+        raise OmnikInverterPacketInvalidError("Invalid end byte")
+
+    expected_checksum = message.pop()
+    checksum = sum(message) & 0xFF
+    if expected_checksum != checksum:
+        raise OmnikInverterPacketInvalidError(
+            f"Checksum mismatch (got `{checksum}` expected `{expected_checksum}`"
+        )
+
+    return message
+
+
+def create_information_request(serial_number: int) -> bytearray:
+    """Compute a magic message to which the Omnik will reply with raw statistics.
+
+    Args:
+        serial_number: Integer with the serial number of your Omnik device.
+
+    Returns:
+        A bytearray with the raw message data, to be sent over a TCP socket.
+    """
+    request_data = bytearray([0x02, 0x40, 0x30])
+    serial_bytes = serial_number.to_bytes(length=4, byteorder="little")
+    request_data.extend(serial_bytes)
+    request_data.extend(serial_bytes)
+    request_data.extend([0x01, 0x00])
+    return _pack_message(request_data)
+
+
+def parse_information_reply(serial_number: int, data: bytes) -> dict[str, Any]:
+    """Perform a raw TCP request to the Omnik device.
+
+    Args:
+        serial_number: Serial number passed to
+            `clk.create_information_request()`, used to validate the reply.
+        data: Raw data reply from the Omnik Inverter.
+
+    Returns:
+        A Python dictionary (text) with the response from
+        the Omnik Inverter.
+
+    Raises:
+        OmnikInverterPacketInvalidError: Received data fails basic validity checks.
+    """
+
+    data = _unpack_message(bytearray(data))
+    data = _TcpData.from_buffer_copy(data)
+
+    if any(
+        serial_number != int.from_bytes(b, byteorder="little")
+        for b in data.double_serial_number
+    ):
+        raise OmnikInverterPacketInvalidError("Serial number mismatch in reply")
+
+    if data.unknown0 != UINT16_MAX:
+        print(f"Unexpected unknown0 `{data.unknown0}`")
+
+    if data.padding0 != b"\x81\x02\x01":
+        print(f"Unexpected padding0 `{data.padding0}`")
+
+    # For all data that's expected to be zero, print it if it's not. Perhaps
+    # there are more interesting fields on different inverters waiting to be
+    # uncovered.
+    for idx in range(1, 5):
+        name = f"padding{idx}"
+        padding = getattr(data, name)
+        if sum(padding):
+            print(f"Unexpected `{name}`: `{padding}`")
+
+    def extract_model(magic: c_char * 3):
+        return {
+            b"\x7d\x41\xb0": "Omniksol 3000TL",
+            # http://www.mb200d.nl/wordpress/2015/11/omniksol-4k-tl-wifi-kit/#more-590:
+            b"\x81\x41\xb0": "Omniksol 4000TL",
+            b"\x9a\x41\xb0": "Omniksol 4000TL2",
+        }.get(magic, f"Unknown device model from magic {magic!r}")
+
+    def list_divide_10(integers: list[c_ushort]) -> list[c_ushort]:
+        return [None if v == UINT16_MAX else v * 0.1 for v in integers]
+
+    def int_to_bool(num: c_ushort):
+        return {
+            0: False,
+            1: True,
+        }[num]
+
+    # Only these fields will be extracted from the structure
+    field_extractors = {
+        "model": extract_model,
+        "serial_number": None,
+        "temperature": 0.1,
+        "dc_input_voltage": list_divide_10,
+        "dc_input_current": list_divide_10,
+        "ac_output_current": list_divide_10,
+        "ac_output_voltage": list_divide_10,
+        "ac_output": None,
+        "solar_energy_today": 0.01,
+        "solar_energy_total": 0.1,
+        "solar_hours_total": None,
+        "inverter_active": int_to_bool,
+        "firmware": None,
+        "firmware_slave": None,
+    }
+
+    result = {}
+
+    for (name, extractor) in field_extractors.items():
+        value = getattr(data, name)
+        if name == "ac_output":
+            # Flatten the list of frequency+power AC objects
+
+            result["ac_output_frequency"] = [o.get_frequency() for o in value]
+            result["ac_output_power"] = [o.get_power() for o in value]
+            continue
+
+        if isinstance(extractor, float):
+            value *= extractor
+        elif extractor is not None:
+            value = extractor(value)
+
+        result[name] = value
+
+    return result