Added nodedata and google graph test page

deploy/micropython/esp8266/deploy-dev.cmd

@@ -1,3 +1,3 @@
 @echo off
 rem python -m esptool --port %1 erase_flash
-python -m esptool --chip esp8266 --port %1 --baud 921600 --before default_reset --after hard_reset write_flash --verify --flash_size=detect --flash_mode=qio 0 v1.9.4-568-g4df194394-dirty-2018-10-04.bin
+python -m esptool --chip esp8266 --port %1 --baud 921600 --before default_reset --after hard_reset write_flash --verify --flash_size=detect --flash_mode=qio 0 v1.9.4-568-g4df194394-dirty-2018-10-08.bin

src/core/prometheus/nodedata.py

@@ -0,0 +1,201 @@
+# coding=utf-8
+"""
+server - recv packet (buffer), parse and validate node to a-z, write to node/sensor/%yy.%mm.%dd/data.json:
+record:
+[
+  {
+      'timestamp': 'iso-timestamp',
+      data: [
+          {'sensor': value},
+          (..)
+      ]
+  }
+]
+client - send list of packets
+packet: node\tsensor\tvalue\r\n
+function to parse data file into individual readings again?
+"""
+import socket
+import time
+import gc
+import os
+import prometheus
+import prometheus.psocket
+import prometheus.logging
+
+
+__version__ = '0.1.0'
+__author__ = 'Hans Christian Winther-Sorensen'
+
+gc.collect()
+
+
+class NodeData:
+    """
+NodeData storage and formatting class
+    """
+    def __init__(self, node, sensor, value, timestamp=None):
+        # type: (bytes, bytes, bytes, int) -> NodeData
+        """
+Data formatting class, raw data to/from json
+        :param node: node/device name
+        :param sensor: sensor name
+        :param value: sensor value
+        :param timestamp: should be left empty unless loading from file
+        """
+        self.node = node
+        self.sensor = sensor
+        self.value = value
+        if timestamp is None:
+            timestamp = time.time()
+        self.timestamp = timestamp
+
+    def to_json(self):
+        """
+Convert to JSON format
+        :return: JSON as string
+        """
+        import json
+        return json.dumps({'node': self.node, 'sensor': self.sensor, 'value': self.value, 'timestamp': self.timestamp})
+
+    @staticmethod
+    def from_json(data):
+        """
+Construct NodeData instance from JSON format
+        :param data: JSON object
+        :return: NodeData instance
+        """
+        return NodeData(data['node'], data['sensor'], data['value'], data['timestamp'])
+
+    def to_packet(self):
+        """
+Convert to packet (raw) format
+        :return: bytes
+        """
+        return b'%s\t%s\t%s' % (self.node, self.sensor, self.value)
+
+    @staticmethod
+    def from_packet(data):
+        """
+Construct NodeData instance from packet (raw) format
+        :param data: bytes
+        :return: NodeData instance
+        """
+        parts = data.split(b'\t')
+        print(parts, len(parts))
+        if len(parts) != 3:
+            return None
+        node_data = NodeData(parts[0], parts[1], parts[2])
+        import datetime
+        node_data.timestamp = datetime.datetime.now().isoformat()
+        return node_data
+
+
+class Server:
+    """
+NodeData UDP server
+    """
+    def start(self, bind_host='', bind_port=9085):
+        """
+NodeData UDP receiver
+        :param bind_host: IP to bind to, defaults to any
+        :param bind_port: Port to bind to, defaults to 9085
+        """
+        sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+        sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+        sock.bind((bind_host, bind_port))
+        prometheus.logging.info('Listening on %s:%d' % (bind_host, bind_port))
+        sock.settimeout(0.1)
+        buffers = dict()
+        split_chars = '\n'
+        end_chars = '\r'
+        debug = True
+
+        while True:
+            data, addr = None, None
+            try:
+                data, addr = sock.recvfrom(500)
+            except prometheus.psocket.socket_error as exception:
+                if prometheus.is_micro:
+                    if exception.args[0] != 11 and exception.args[0] != 110 and exception.args[0] != 23:
+                        prometheus.logging.error(exception)
+                        raise
+                else:
+                    if exception.errno != 11 and exception.errno != 110 and exception.errno != 10035 and \
+                            not isinstance(exception, socket.timeout):
+                        prometheus.logging.error(exception)
+                        raise
+
+            if data is None:
+                continue
+
+            prometheus.logging.notice('recv %s from %s' % (repr(data), repr(addr)))
+
+            if addr not in buffers.keys():
+                if debug:
+                    prometheus.logging.debug('Creating new buffer context')
+                buffers[addr] = prometheus.Buffer(split_chars=split_chars, end_chars=end_chars)
+
+            buffers[addr].parse(data)
+
+            while True:
+                command = buffers[addr].pop()
+                if command is None:
+                    break
+
+                prometheus.logging.debug('handle_packet')
+                self.handle_packet(addr, command)
+
+    @staticmethod
+    def handle_packet(addr, command):
+        """
+Append packet to file
+        :param addr: (host, port) tuple
+        :param command: command data
+        :return: None
+        """
+        packet = NodeData.from_packet(command.packet)
+        if packet is None:
+            return
+        prometheus.logging.info('addr=%s cmd=%s' % (addr, command))
+        import datetime
+        date_time = datetime.datetime.now()
+        filename = '%s.json' % date_time.strftime('%Y.%m.%d')
+        folder = packet.node
+        if not os.path.exists('data'):
+            os.mkdir('data')
+        folder = os.path.join('data', folder)
+        if not os.path.exists(folder):
+            os.mkdir(folder)
+        filepath = os.path.join(folder, filename)
+        prometheus.logging.info('writing to %s' % filepath)
+        with open(filepath, 'a') as file_descriptor:
+            file_descriptor.write(packet.to_json() + b'\r\n')
+
+
+def client(host, port, *nodedatas):
+    """
+NodeData sending client
+    :param host: Host to connect to
+    :param port: Port to connect to
+    :param nodedatas: List of NodeData instances
+    """
+    sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+    for nodedata in nodedatas:
+        assert isinstance(nodedata, NodeData)
+        prometheus.logging.info('Sending %s=%s' % (nodedata.sensor, nodedata.value))
+        sock.sendto(nodedata.to_packet() + b'\r\n', (host, port))
+        gc.collect()
+    gc.collect()
+
+
+def run_server():
+    """
+Start UDP server instance
+    """
+    server = Server()
+    server.start()
+
+
+if __name__ == '__main__':
+    run_server()

src/devices/esp8266-nodetest/deploy.py

@@ -10,8 +10,8 @@
 '''
 
 files = [
-    # 'main.py',
-    'nodetest.py',
+    'main.py',
+    # 'nodetest.py',
 ]
 # prometheus.tftpd.tftp_client('nodetest.iot.oh.wsh.no', *files)
 prometheus.tftpd.tftp_client('10.20.2.117', *files)

src/devices/esp8266-nodetest/main.py

@@ -35,11 +35,11 @@
 logging.boot(tcpserver)
 gc.collect()
 logging.debug('mem_free before start: %d' % gc.mem_free())
-# try:
-tcpserver.start()
-# except Exception as exception:
-#     print(exception)
-#     gc.collect()
+try:
+    tcpserver.start()
+except Exception as exception:
+    print(exception)
+    gc.collect()
 
 try:
     logging.error('crashed? rst')

test/clients/localtester.py

@@ -7,6 +7,14 @@
 # from deploy.clients import proxytest2client
 import prometheus.logging as logging
 import time
+import prometheus.nodedata
+import sys
+
+
+packets = list()
+packets.append(prometheus.nodedata.NodeData(b'testnode', b'testsensor', b'testvalue'))
+prometheus.nodedata.client('localhost', 9085, *packets)
+sys.exit(0)
 
 
 class StressTester(object):

test/graphtest/gendata.py

@@ -0,0 +1,40 @@
+# coding=utf-8
+import os
+import json
+import dateutil.parser
+
+path = '/srv/nodedata/data'
+d = dict()
+for folder in os.listdir(path):
+    folder_path = os.path.join(path, folder)
+    # records = list()
+    for file in os.listdir(folder_path):
+        file_path = os.path.join(folder_path, file)
+        for line in open(file_path, 'r').read().replace('\r', '').split('\n'):
+            if line == '':
+                continue
+            jobject = json.loads(line)
+            # records.append(jobject)
+            dt = dateutil.parser.parse(jobject['timestamp'])
+            key = dt.strftime('%Y.%m.%d-%H:%M:%S')
+            if key not in d:
+                d[key] = list()
+            d[key].append(jobject)
+
+lst = list()
+keys = d.keys()
+keys.sort()
+for key in keys:
+    values = d[key]
+    # print(values)
+    d3 = dict()
+    d3['timestamp'] = values[0]['timestamp']
+    for value in values:
+        name = value['node'] + '_' + value['sensor']
+        # print(repr(name))
+        d3[name] = float(value['value'])
+    lst.append(d3)
+
+js_filepath = '/srv/nodedata/html/sensors.js'
+print('writing to %s' % js_filepath)
+open(js_filepath, 'w').write('window.sensor_data = ' + json.dumps(lst) + '\r\n')

test/graphtest/nodedatatest.html

@@ -0,0 +1,54 @@
+<html>
+<head>
+    <script type="text/javascript" src="https://www.gstatic.com/charts/loader.js"></script>
+    <script type="text/javascript" src="sensors.js"></script>
+    <script type="text/javascript">
+        google.charts.load('current', {'packages': ['corechart']});
+        google.charts.setOnLoadCallback(drawChart);
+
+        function drawChart() {
+
+            var data = new google.visualization.DataTable();
+            data.addColumn('datetime', 'Time');
+            data.addColumn('number', 'Indoor temperature');
+            data.addColumn('number', 'Indoor humidity');
+            data.addColumn('number', 'Outdoor temperature');
+            data.addColumn('number', 'Outdoor humidity');
+
+            for(var i = 0; i < window.sensor_data.length; i++) {
+                var item = window.sensor_data[i];
+                //console.log(item['timestamp']);
+                //console.log(new Date(item['timestamp']));
+
+                data.addRow([
+                    new Date(item['timestamp']), item['sensor01_dht_temp'], item['sensor01_dht_hum'], item['sensor02_dht_temp'], item['sensor02_dht_hum']
+                ]);
+            }
+
+            /*
+            var options = {
+                chart: {
+                    title: 'IOT',
+                    subtitle: 'sensor data'
+                },
+                //width: 900,
+                height: 800
+            };
+            var chart = new google.charts.Line(document.getElementById('chart_div'));
+            chart.draw(data, google.charts.Line.convertOptions(options));
+            */
+
+            var options = {
+                title: 'IOT sensor data',
+                hAxis: {title: 'Time',  titleTextStyle: {color: '#333'}},
+                vAxis: {minValue: 0}
+            };
+        var chart = new google.visualization.AreaChart(document.getElementById('chart_div'));
+        chart.draw(data, options);
+        }
+    </script>
+</head>
+<body>
+<div id="chart_div" style="width: 100%; height: 800px"></div>
+</body>
+</html>