arne/sdlogger_old
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

sdlogger working upload and logging

Browse source
This commit is contained in:
Arne Zachlod 2025-02-20 18:04:19 +01:00
parent 920d7619be
commit 2a20349bd2
2 changed files with 171 additions and 32 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

28
README.md
View file

@ -1,3 +1,31 @@
# SDLOGGER
sdlogger takes all data from uart0 rx and writes it onto a sdcard.
# available commands
- RTC
- "epoch" : unix timestamp in seconds
- upload
- wlan_ssid
- wlan_password: optional, do not provide if open wlan is to be used
- upload_server: server URL, e.g.: "http://localhost" port 80/443 will get autoassigned based on http or https
- router_mac: mac address of the router, will be used in filename
- UOTA
- wlan_ssid
- wlan_password: optional, do not provide if open wlan is to be used
- reset
- no options
## creating JSON for commands
```
#! /usr/bin/env python
import json
json.dumps({'cmd': 'RTC','epoch':123456778890})
```

173
sdlogger.py
View file

@ -1,30 +1,41 @@
# SPDX-License-Identifier: GPL-2.0-or-later
BUFSIZE_BUFFER = const(49 * 1024)
BUFSIZE_RXBUF = const(32 * 1024)
BUF_THRESHOLD = const(16 * 1024)
UART_TIMEOUT = const(1000) # timeout to start writing in ms
# buffer is very big chunk of memory, therefore we allocate it as early as possible
BUFFER = bytearray(BUFSIZE_BUFFER)
import os, vfs
import time
import json
import gc
from machine import UART
from machine import SPI
from machine import Pin
from machine import RTC
from machine import WDT
from machine import reset
from sdcard import SDCard
DEBUG = True # set True for debug output
BUF_POS = 0
SD_MOUNT = '/sd'
LOG_FOLDER = 'logs'
OLD_LOG_FOLDER = 'old_logs'
LOG_FILENAME = 'logfile.log'
LOG_PATH = SD_MOUNT + '/' + LOG_FOLDER + '/' + LOG_FILENAME
CMD_PREFIX = const(b"sdlogger {")
CMD_PREFIX = const(b"__sdlogger__ {")
BUF_LASTLINE = bytearray(1024)
BUF_POS_LASTLINE = 0
WRITETIME = time.ticks_ms()
# access token for demo server
URL_ACCESS_TOKEN = 'Ti0TahcaiN0Ahkeb1eegaiv6gu'
UART0 = UART(0, baudrate=115200, rx=20, tx=21, rxbuf=BUFSIZE_RXBUF)
SDSPI = SPI(1,
@ -37,14 +48,22 @@ SDSPI = SPI(1,
mosi=Pin(2),
miso=Pin(4))
SD = SDCard(SDSPI, Pin(9, Pin.OUT))
RTC0 = RTC()
def debug(text):
global DEBUG
if DEBUG == True:
print("DEBUG: " + text)
def sdcard_init():
"""sdcard setup, mounting and creating logfile and folder"""
os.mount(SD, SD_MOUNT)
dirls = os.listdir(SD_MOUNT)
if not LOG_FOLDER in dirls:
os.mkdir(SD_MOUNT + '/' + LOG_FOLDER)
if not OLD_LOG_FOLDER in dirls:
os.mkdir(SD_MOUNT + '/' + OLD_LOG_FOLDER)
# find the new filename for LOG_FILENAME. eg: foo.log23
dirls = os.listdir(SD_MOUNT + '/' + LOG_FOLDER)
x = -1
@ -64,14 +83,13 @@ def writebuf():
global BUF_POS
global WRITETIME
global LOG_PATH
dbg_tmr = time.ticks_ms()
global DEBUG
fd = open(LOG_PATH, "ab")
mv = memoryview(BUFFER)
fd.write(mv[:BUF_POS])
fd.flush()
fd.close()
diff_tmr = time.ticks_diff(time.ticks_ms(), dbg_tmr)
print(f"SD write time: {diff_tmr} ms for {BUF_POS} bytes")
debug("SD write: " + str(BUF_POS) + " bytes")
BUF_POS = 0
WRITETIME = time.ticks_ms()
@ -99,66 +117,94 @@ def parse_cmd():
# search for CMD_PREFIX between first and last newline
#
# lastline = from last newline
# linux kernel sends \r\n on line end
global BUF_POS
global BUF_POS_LASTLINE
global BUF_LASTLINE
global BUFFER
firstnl = BUFFER.find(b'\r', 0, BUF_POS)
if (firstnl != -1) and ((firstnl + BUF_POS_LASTLINE) < len(BUF_LASTLINE)):
debug("parse_cmd start")
debug("BUF_POS: " + str(BUF_POS))
firstnl = BUFFER.find(b'\r\n', 0, BUF_POS)
debug("firstnl: " + str(firstnl))
# if firstnl == -1: line not complete (yet)
# append whole line to BUF_LASTLINE
# add bytes written to BUF_POS_LASTLINE
if firstnl == -1:
# write part of line into BUF_LASTLINE
buf_line_len = BUF_POS_LASTLINE + BUF_POS
BUF_LASTLINE[BUF_POS_LASTLINE:buf_line_len] = BUFFER[0:BUF_POS]
BUF_POS_LASTLINE = buf_line_len
return
# found a newline, will append it to BUF_LASTLINE and check for command
if (firstnl != -1) and (firstnl + BUF_POS_LASTLINE) < len(BUF_LASTLINE):
buf_line_len = BUF_POS_LASTLINE + firstnl
BUF_LASTLINE[BUF_POS_LASTLINE:buf_line_len] = BUFFER[0:firstnl]
cmd = BUF_LASTLINE.find(CMD_PREFIX, 0, buf_line_len)
# found command in BUF_LASTLINE
if (cmd != -1):
json_start = cmd + len(CMD_PREFIX) - 1
mv = memoryview(BUF_LASTLINE)
exec_cmd(mv[json_start:buf_line_len])
lastnl = BUFFER.rfind(b'\r', BUF_POS)
t = BUF_POS - lastnl
lastnl = BUFFER.rfind(b'\r\n', firstnl, BUF_POS)
debug("lastnl: " + str(lastnl))
# found last newline, will copy into BUF_LASTLINE from lastnl till BUF_POS
# update BUF_POS_LASTLINE
# this is always the start of a BUF_LASTLINE, therefore implicis reset of BUF_POS_LASTLINE
if lastnl > 0:
t = BUF_POS - (lastnl + 2)
debug("t: " + str(t))
BUF_LASTLINE[0:t] = BUFFER[lastnl:BUF_POS]
cmd = 0
while cmd != -1:
cmd = BUFFER.find(CMD_PREFIX, firstnl, lastnl)
if cmd != -1:
line_end = BUFFER.find(b'\r', cmd, lastnl)
json_start = cmd + len(CMD_PREFIX) - 1
BUF_POS_LASTLINE = t
# check for command between firstnl and lastnl
cmd_pos = firstnl
while True:
cmd_pos = BUFFER.find(CMD_PREFIX, cmd_pos, lastnl)
if cmd_pos == -1:
break
else:
line_end = BUFFER.find(b'\r', cmd_pos, lastnl)
cmd_pos = cmd_pos + len(CMD_PREFIX) - 1
mv = memoryview(BUFFER)
exec_cmd(mv[json_start:line_end])
exec_cmd(mv[cmd_pos:line_end])
def exec_cmd(json_cmd):
try:
cmd = json.loads(json_cmd)
except:
except Exception:
return
if cmd['cmd'] == "rtc":
debug("command found: " + cmd['cmd'])
if cmd['cmd'] == "RTC":
exec_rtc(cmd)
elif cmd['cmd'] == "upload":
exec_upload(cmd)
elif cmd['cmd'] == "UOTA":
exec_uota(cmd)
elif cmd['cmd'] == "reset":
exec_reset(cmd)
def exec_rtc(cmd):
rtc = machine.RTC()
# the command should have an option "epoch", giving seconds since year 2000
rtc.datetime(time.gmtime(cmd['epoch']))
global RTC0
# the command should have an option "epoch", giving seconds since year 1970
# micropython uses epoch based in 2000, so 946684800 seconds later
RTC0.datetime(time.gmtime(cmd['epoch'] - 946684800))
debug(str(RTC0.datetime()))
def exec_reset(cmd):
import machine
machine.reset()
reset()
def exec_upload(cmd):
# options:
# wlan_ssid
# wlan_password - optional
# upload_server
global BUFFER
del BUFFER # delete buffer to free some memory for upload process
def network_connect(cmd):
if 'wlan_password' in cmd:
pass
else:
cmd['wlan_password'] = None
# connect to WLAN
debug("connecting to ssid " + str(cmd['wlan_ssid']) + " password: " + str(cmd['wlan_password']))
import network
wlan = network.WLAN(network.WLAN.IF_STA)
wlan.active(True)
@ -166,8 +212,64 @@ def exec_upload(cmd):
wlan.connect(cmd['wlan_ssid'], cmd['wlan_password'])
while not wlan.isconnected():
pass
# TODO: upload file to server
debug("wlan connected")
def exec_upload(cmd):
# options:
# wlan_ssid
# wlan_password - optional
# upload_server
# router_mac - MACaddress of the router
global BUFFER
global URL_ACCESS_TOKEN
global LOG_FOLDER
global OLD_LOG_FOLDER
del BUFFER # delete buffer to free some memory for upload process
gc.collect()
debug(f"free memory before network connect: {gc.mem_free()}")
network_connect(cmd)
# upload file to server
import requests
logpath = SD_MOUNT + '/' + LOG_FOLDER
old_logpath = SD_MOUNT + '/' + OLD_LOG_FOLDER
ls = os.listdir(logpath)
debug("files to upload: " + str(ls))
for file in ls:
gc.collect()
response = requests.request("POST", cmd['upload_server'] + '/' + cmd['router_mac'] + file, data=sd_read_chunks(logpath + '/' + file), timeout=60, headers={"Access-Token": URL_ACCESS_TOKEN})
debug(f"HTTP status code: {response.status_code}")
if response.status_code == 200:
# move uploaded file to OLD_LOG_FOLDER
os.rename(logpath + '/' + file, old_logpath + '/' + file)
pass
# hard reset
reset()
def sd_read_chunks(file):
fd = open(file, "rb")
buf = bytearray(512)
mv = memoryview(buf)
while True:
length = fd.readinto(mv)
debug(f"read {length} bytes {fd.tell()}, garbage: {gc.mem_free()}")
yield mv[0:length - 1]
if length < len(buf):
break
fd.close()
debug("read file " + str(file) + " from SD")
def exec_uota(cmd):
# options:
# wlan_ssid
# wlan_password - optional
global BUFFER
del BUFFER
gc.collect()
import uota
network_connect(cmd)
if uota.check_for_updates():
uota.install_new_firmware()
reset()
def control():
"""main control loop"""
@ -181,8 +283,17 @@ def control():
def main():
sdcard_init()
debug("SD initialized, entering control loop")
while True:
control()
main()
if not DEBUG:
try:
main()
except Exception:
# dont catch e.g. KeyboardInterrupt or SystemExit
reset()
else:
main()