diff --git a/README.md b/README.md index 0f70eea..b05fd9f 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,31 @@ # SDLOGGER sdlogger takes all data from uart0 rx and writes it onto a sdcard. + + + +# available commands + +- RTC + - "epoch" : unix timestamp in seconds +- upload + - wlan_ssid + - wlan_password: optional, do not provide if open wlan is to be used + - upload_server: server URL, e.g.: "http://localhost" port 80/443 will get autoassigned based on http or https + - router_mac: mac address of the router, will be used in filename +- UOTA + - wlan_ssid + - wlan_password: optional, do not provide if open wlan is to be used +- reset + - no options + +## creating JSON for commands + +``` +#! /usr/bin/env python +import json + +json.dumps({'cmd': 'RTC','epoch':123456778890}) + +``` + diff --git a/sdlogger.py b/sdlogger.py index 12b30f7..92559c4 100755 --- a/sdlogger.py +++ b/sdlogger.py @@ -1,30 +1,41 @@ # SPDX-License-Identifier: GPL-2.0-or-later + + BUFSIZE_BUFFER = const(49 * 1024) BUFSIZE_RXBUF = const(32 * 1024) BUF_THRESHOLD = const(16 * 1024) UART_TIMEOUT = const(1000) # timeout to start writing in ms +# buffer is very big chunk of memory, therefore we allocate it as early as possible BUFFER = bytearray(BUFSIZE_BUFFER) import os, vfs import time import json +import gc from machine import UART from machine import SPI from machine import Pin from machine import RTC +from machine import WDT +from machine import reset from sdcard import SDCard +DEBUG = True # set True for debug output + BUF_POS = 0 SD_MOUNT = '/sd' LOG_FOLDER = 'logs' +OLD_LOG_FOLDER = 'old_logs' LOG_FILENAME = 'logfile.log' LOG_PATH = SD_MOUNT + '/' + LOG_FOLDER + '/' + LOG_FILENAME -CMD_PREFIX = const(b"sdlogger {") +CMD_PREFIX = const(b"__sdlogger__ {") BUF_LASTLINE = bytearray(1024) BUF_POS_LASTLINE = 0 WRITETIME = time.ticks_ms() +# access token for demo server +URL_ACCESS_TOKEN = 'Ti0TahcaiN0Ahkeb1eegaiv6gu' UART0 = UART(0, baudrate=115200, rx=20, tx=21, rxbuf=BUFSIZE_RXBUF) SDSPI = SPI(1, @@ -37,14 +48,22 @@ SDSPI = SPI(1, mosi=Pin(2), miso=Pin(4)) SD = SDCard(SDSPI, Pin(9, Pin.OUT)) +RTC0 = RTC() +def debug(text): + global DEBUG + if DEBUG == True: + print("DEBUG: " + text) + def sdcard_init(): """sdcard setup, mounting and creating logfile and folder""" os.mount(SD, SD_MOUNT) dirls = os.listdir(SD_MOUNT) if not LOG_FOLDER in dirls: os.mkdir(SD_MOUNT + '/' + LOG_FOLDER) + if not OLD_LOG_FOLDER in dirls: + os.mkdir(SD_MOUNT + '/' + OLD_LOG_FOLDER) # find the new filename for LOG_FILENAME. eg: foo.log23 dirls = os.listdir(SD_MOUNT + '/' + LOG_FOLDER) x = -1 @@ -64,14 +83,13 @@ def writebuf(): global BUF_POS global WRITETIME global LOG_PATH - dbg_tmr = time.ticks_ms() + global DEBUG fd = open(LOG_PATH, "ab") mv = memoryview(BUFFER) fd.write(mv[:BUF_POS]) fd.flush() fd.close() - diff_tmr = time.ticks_diff(time.ticks_ms(), dbg_tmr) - print(f"SD write time: {diff_tmr} ms for {BUF_POS} bytes") + debug("SD write: " + str(BUF_POS) + " bytes") BUF_POS = 0 WRITETIME = time.ticks_ms() @@ -99,66 +117,94 @@ def parse_cmd(): # search for CMD_PREFIX between first and last newline # # lastline = from last newline + # linux kernel sends \r\n on line end global BUF_POS global BUF_POS_LASTLINE global BUF_LASTLINE global BUFFER - firstnl = BUFFER.find(b'\r', 0, BUF_POS) - if (firstnl != -1) and ((firstnl + BUF_POS_LASTLINE) < len(BUF_LASTLINE)): + debug("parse_cmd start") + + debug("BUF_POS: " + str(BUF_POS)) + firstnl = BUFFER.find(b'\r\n', 0, BUF_POS) + debug("firstnl: " + str(firstnl)) + + # if firstnl == -1: line not complete (yet) + # append whole line to BUF_LASTLINE + # add bytes written to BUF_POS_LASTLINE + if firstnl == -1: + # write part of line into BUF_LASTLINE + buf_line_len = BUF_POS_LASTLINE + BUF_POS + BUF_LASTLINE[BUF_POS_LASTLINE:buf_line_len] = BUFFER[0:BUF_POS] + BUF_POS_LASTLINE = buf_line_len + return + + # found a newline, will append it to BUF_LASTLINE and check for command + if (firstnl != -1) and (firstnl + BUF_POS_LASTLINE) < len(BUF_LASTLINE): buf_line_len = BUF_POS_LASTLINE + firstnl BUF_LASTLINE[BUF_POS_LASTLINE:buf_line_len] = BUFFER[0:firstnl] cmd = BUF_LASTLINE.find(CMD_PREFIX, 0, buf_line_len) + # found command in BUF_LASTLINE if (cmd != -1): json_start = cmd + len(CMD_PREFIX) - 1 mv = memoryview(BUF_LASTLINE) exec_cmd(mv[json_start:buf_line_len]) - lastnl = BUFFER.rfind(b'\r', BUF_POS) - t = BUF_POS - lastnl - BUF_LASTLINE[0:t] = BUFFER[lastnl:BUF_POS] - cmd = 0 - while cmd != -1: - cmd = BUFFER.find(CMD_PREFIX, firstnl, lastnl) - if cmd != -1: - line_end = BUFFER.find(b'\r', cmd, lastnl) - json_start = cmd + len(CMD_PREFIX) - 1 + lastnl = BUFFER.rfind(b'\r\n', firstnl, BUF_POS) + debug("lastnl: " + str(lastnl)) + # found last newline, will copy into BUF_LASTLINE from lastnl till BUF_POS + # update BUF_POS_LASTLINE + # this is always the start of a BUF_LASTLINE, therefore implicis reset of BUF_POS_LASTLINE + if lastnl > 0: + t = BUF_POS - (lastnl + 2) + debug("t: " + str(t)) + BUF_LASTLINE[0:t] = BUFFER[lastnl:BUF_POS] + BUF_POS_LASTLINE = t + # check for command between firstnl and lastnl + cmd_pos = firstnl + while True: + cmd_pos = BUFFER.find(CMD_PREFIX, cmd_pos, lastnl) + if cmd_pos == -1: + break + else: + line_end = BUFFER.find(b'\r', cmd_pos, lastnl) + cmd_pos = cmd_pos + len(CMD_PREFIX) - 1 mv = memoryview(BUFFER) - exec_cmd(mv[json_start:line_end]) + exec_cmd(mv[cmd_pos:line_end]) def exec_cmd(json_cmd): try: cmd = json.loads(json_cmd) - except: + except Exception: return - if cmd['cmd'] == "rtc": + debug("command found: " + cmd['cmd']) + if cmd['cmd'] == "RTC": exec_rtc(cmd) elif cmd['cmd'] == "upload": exec_upload(cmd) + elif cmd['cmd'] == "UOTA": + exec_uota(cmd) elif cmd['cmd'] == "reset": exec_reset(cmd) def exec_rtc(cmd): - rtc = machine.RTC() - # the command should have an option "epoch", giving seconds since year 2000 - rtc.datetime(time.gmtime(cmd['epoch'])) + global RTC0 + # the command should have an option "epoch", giving seconds since year 1970 + # micropython uses epoch based in 2000, so 946684800 seconds later + RTC0.datetime(time.gmtime(cmd['epoch'] - 946684800)) + debug(str(RTC0.datetime())) def exec_reset(cmd): - import machine - machine.reset() + reset() -def exec_upload(cmd): - # options: - # wlan_ssid - # wlan_password - optional - # upload_server - global BUFFER - del BUFFER # delete buffer to free some memory for upload process +def network_connect(cmd): if 'wlan_password' in cmd: pass else: cmd['wlan_password'] = None + # connect to WLAN + debug("connecting to ssid " + str(cmd['wlan_ssid']) + " password: " + str(cmd['wlan_password'])) import network wlan = network.WLAN(network.WLAN.IF_STA) wlan.active(True) @@ -166,8 +212,64 @@ def exec_upload(cmd): wlan.connect(cmd['wlan_ssid'], cmd['wlan_password']) while not wlan.isconnected(): pass - # TODO: upload file to server + debug("wlan connected") +def exec_upload(cmd): + # options: + # wlan_ssid + # wlan_password - optional + # upload_server + # router_mac - MACaddress of the router + global BUFFER + global URL_ACCESS_TOKEN + global LOG_FOLDER + global OLD_LOG_FOLDER + del BUFFER # delete buffer to free some memory for upload process + gc.collect() + debug(f"free memory before network connect: {gc.mem_free()}") + network_connect(cmd) + # upload file to server + import requests + logpath = SD_MOUNT + '/' + LOG_FOLDER + old_logpath = SD_MOUNT + '/' + OLD_LOG_FOLDER + ls = os.listdir(logpath) + debug("files to upload: " + str(ls)) + for file in ls: + gc.collect() + response = requests.request("POST", cmd['upload_server'] + '/' + cmd['router_mac'] + file, data=sd_read_chunks(logpath + '/' + file), timeout=60, headers={"Access-Token": URL_ACCESS_TOKEN}) + debug(f"HTTP status code: {response.status_code}") + if response.status_code == 200: + # move uploaded file to OLD_LOG_FOLDER + os.rename(logpath + '/' + file, old_logpath + '/' + file) + pass + # hard reset + reset() + +def sd_read_chunks(file): + fd = open(file, "rb") + buf = bytearray(512) + mv = memoryview(buf) + while True: + length = fd.readinto(mv) + debug(f"read {length} bytes {fd.tell()}, garbage: {gc.mem_free()}") + yield mv[0:length - 1] + if length < len(buf): + break + fd.close() + debug("read file " + str(file) + " from SD") + +def exec_uota(cmd): + # options: + # wlan_ssid + # wlan_password - optional + global BUFFER + del BUFFER + gc.collect() + import uota + network_connect(cmd) + if uota.check_for_updates(): + uota.install_new_firmware() + reset() def control(): """main control loop""" @@ -181,8 +283,17 @@ def control(): def main(): sdcard_init() + debug("SD initialized, entering control loop") while True: control() -main() + +if not DEBUG: + try: + main() + except Exception: + # dont catch e.g. KeyboardInterrupt or SystemExit + reset() +else: + main()