solarrd/update.py

#!/usr/bin/env python3
import sys
import os
import time
import serial
import crcmod
import rrdtool
import paramiko

from time import sleep

from config import *

calc_crc = crcmod.predefined.mkCrcFun(CRC_TYPE)


def parse_line(line):
    if line[-2:] != "\r\n":
        print("Failed to parse: newline broken")
        return {key: None for key in FORMAT}
    line = line[:-2]

    crc_str = line[-4:]
    payload = line[0:-4]
    crc = int(crc_str, 16)
    calced_crc = calc_crc(payload.encode('ascii'))
    if crc != calc_crc(payload.encode('ascii')):
        print("Failed to parse: CRC broken")
        return {key: None for key in FORMAT}

    parts = payload.split(';')
    parts = [p.strip() for p in parts]
    parts = [x if x != '#' else None for x in parts]
    data = zip(FORMAT, parts)

    return dict(data)


def create_database():
    sources = []
    for name, minval, maxval in STORED_VALUES:
        sources += [
            "DS:%s:GAUGE:%d:%f:%f" % (name, DATA_INTERVAL * 2, minval, maxval)
        ]

    now = time.time()
    now = now - (now % DATA_INTERVAL)

    rrd_params = [
        DATA_FILE, "--start",
        "%d" % now, "--step",
        str(DATA_INTERVAL)
    ]
    rrd_params += sources
    rows = ARCHIVE_INTERVAL / DATA_INTERVAL
    rrd_params += ["RRA:LAST:0.1:1:%d" % (rows, )]
    rrdtool.create(*rrd_params)

    rrd_archive_params = [
        ARCHIVE_DATA_FILE, "--start", "now", "--step",
        str(DATA_INTERVAL)
    ]
    steps = ARCHIVE_INTERVAL / DATA_INTERVAL
    rows = ARCHIVE_KEEP_INTERVAL / ARCHIVE_INTERVAL
    rrd_archive_params += sources
    rrd_archive_params += ["RRA:AVERAGE:%f:%d:%d" % (MAX_MISSING, steps, rows)]
    rrdtool.create(*rrd_archive_params)


def update_database(line):
    update_values = []
    for name, _, _ in STORED_VALUES:

        if line[name] != None:
            update_values += ["%f" % float(line[name])]
        else:
            update_values += [""]

    now = time.time()
    now = now - (now % DATA_INTERVAL)
    line = ("%d:" % now) + ":".join(update_values)

    try:
        rrdtool.update(DATA_FILE, line)
        rrdtool.update(ARCHIVE_DATA_FILE, line)
    except rrdtool.OperationalError:
        print("Failed updating files")


def update_graphs():
    for graph_name, lines in GRAPHS.items():

        # Render current data
        graph_params = [
            '%s.png' % graph_name, '-a', 'PNG', '-s',
            'n-%d' % ARCHIVE_INTERVAL
        ]

        for name, lable, color in lines:
            graph_params += ['DEF:%s=%s:%s:LAST' % (name, DATA_FILE, name)]
            graph_params += ['LINE1:%s%s:%s' % (name, color, lable)]

        try:
            rrdtool.graph(*graph_params)
        except rrdtool.OperationalError:
            print("Failed to render current data")

        # Also render Archives
        graph_params = [
            '%s_archive.png' % graph_name, '-a', 'PNG', '-s',
            'n-%d' % ARCHIVE_KEEP_INTERVAL
        ]

        for name, lable, color in lines:
            graph_params += ['DEF:%s=%s:%s:AVERAGE' % (name, DATA_FILE, name)]
            graph_params += ['LINE1:%s%s:%s' % (name, color, lable)]

        try:
            rrdtool.graph(*graph_params)
        except rrdtool.OperationalError:
            print("Failed to render archive data")


def _do_upload(sftp, src, dest):
    try:
        sftp.put(src, dest)
    except:
        print("Unexpected error while uploading:", sys.exc_info()[1])


def upload_graphs():

    key = paramiko.RSAKey.from_private_key_file(SFTP_KEY)
    transport = paramiko.Transport((SFTP_HOST, SFTP_PORT))
    transport.connect()
    transport.auth_publickey(SFTP_USER, key)
    sftp = paramiko.SFTPClient.from_transport(transport)

    for name, _ in GRAPHS.items():

        _do_upload(sftp, '%s.png' % name, 'solar/%s.png' % name)
        _do_upload(sftp, '%s_archive.png' % name,
                   'solar/%s_archive.png' % name)

    _do_upload(sftp, 'index.html', 'solar/index.html')
    sftp.close()
    transport.close()


def main():
    ser = serial.Serial(SERIAL, BAUD_RATE, timeout=60.0)

    if not os.path.exists(DATA_FILE) or not os.path.exists(ARCHIVE_DATA_FILE):
        create_database()
        sleep(60)

    while True:
        line = ser.readline().decode('ascii')
        if len(line) > 0:
            print(line)
            parsed = parse_line(line)
            print(parsed)
            update_database(parsed)
            update_graphs()
            upload_graphs()
        else:
            print("Timed out, supecting broken usb")
            sys.stdout.flush()
            sys.exit(-1)

        sys.stdout.flush()


if __name__ == '__main__':
    main()