Exergie/DataFlux
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added buffer autosave and fixed slow comports() call by delocalizing to thread

Browse Source
This commit is contained in:
Hector van der Aa
2026-05-18 21:47:02 +02:00
parent e5fd5cda89
commit a69b45ed27
11 changed files with 165 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
src/dataflux/app.py
View File

@@ -2,6 +2,7 @@
# Copyright (C) 2026 Association Exergie <association.exergie@gmail.com>
# SPDX-License-Identifier: GPL-3.0-or-later
from datetime import datetime
from pathlib import Path
import sys
from threading import Thread
@@ -40,6 +41,7 @@ def _asset_path(relative_path: str) -> str:
def run() -> None:
state: AppState = AppState()
state.start_time = datetime.now()
# Create application context and viewport
dpg.create_context()
@@ -91,6 +93,12 @@ def run() -> None:
)
state.telemetry_thread.start()
state.ports_thread_running = True
state.ports_thread = Thread(
target=dataflux.ui.worker.ports_worker, args=(state,), daemon=True
)
state.ports_thread.start()
dpg.start_dearpygui()
dpg.destroy_context()

28
src/dataflux/callbacks/menu.py
View File

@@ -15,6 +15,7 @@ from dataflux.tags import (
GRAPH_X_AXIS_SPEED,
GRAPH_X_AXIS_TENG,
GRAPH_X_AXIS_VBAT,
WINDOW_FILE_DIALOG_AUTOSAVE_BUFFERS,
WINDOW_LORA_CONNECTION_MENU,
WINDOW_FILE_DIALOG_DUMP_BUFFERS,
WINDOW_SERIAL_CONNECTION_MENU,
@@ -27,8 +28,11 @@ def open_lora_connection_window(sender, app_data, user_data: AppState) -> None:
def open_serial_connection_window(sender, app_data, user_data: AppState) -> None:
print("Handling serial window open callback")
dataflux.ui.routines.windows.update_window_serial_connection_menu_combo(user_data)
print("Combo updated")
dpg.show_item(WINDOW_SERIAL_CONNECTION_MENU)
print("Window shown")
def menu_io_disconnect_lora(sender, app_data, user_data: AppState) -> None:
@@ -45,6 +49,10 @@ def menu_file_dump_buffers(sender, app_data, user_data: AppState) -> None:
dpg.show_item(WINDOW_FILE_DIALOG_DUMP_BUFFERS)
def menu_file_quit(sender, app_data, user_data) -> None:
dpg.stop_dearpygui()
def window_file_dialog_dump_buffers_ok(sender, app_data, user_data: AppState) -> None:
user_data.buffer_dump_thread = Thread(
target=dataflux.services.telemetry.buffer_dump,
@@ -54,6 +62,26 @@ def window_file_dialog_dump_buffers_ok(sender, app_data, user_data: AppState) ->
user_data.buffer_dump_thread.start()
def menu_file_autosave_buffers(sender, app_state, user_data: AppState) -> None:
if user_data.autosave_enabled:
user_data.autosave_enabled = False
user_data.autosave_buffer_thread = None
else:
dpg.show_item(WINDOW_FILE_DIALOG_AUTOSAVE_BUFFERS)
def window_file_dialog_autosave_buffers_ok(
sender, app_data, user_data: AppState
) -> None:
user_data.autosave_enabled = True
user_data.autosave_buffer_thread = Thread(
target=dataflux.services.telemetry.autosave_worker,
args=(user_data, app_data["file_path_name"]),
daemon=True,
)
user_data.autosave_buffer_thread.start()
def menu_window_select(sender, app_data, user_data: str) -> None:
dataflux.ui.routines.windows.toggle_window(user_data)

2
src/dataflux/callbacks/serial.py
View File

@@ -8,9 +8,7 @@ import dataflux.ui.routines
from dataflux.state import AppState
from dataflux.tags import (
BUTTON_SERIAL_CONSOLE_SEND,
INPUT_SERIAL_CONSOLE,
TEXT_SERIAL_CONSOLE,
WINDOW_LORA_CONNECTION_MENU,
WINDOW_LORA_CONNECTION_MENU_COMBO,
WINDOW_SERIAL_CONNECTION_MENU,

17
src/dataflux/services/serial/__init__.py
View File

@@ -20,10 +20,11 @@ import dataflux.ui.routines
from dataflux.state import AppState
def list_serial_ports() -> list[str]:
ports = serial.tools.list_ports.comports()
def list_serial_ports(state: AppState) -> list[str]:
valid_ports: list[str] = []
for port in ports:
for port in state.ports:
if port.vid is not None and port.pid is not None:
valid_ports.append(port.device)
@@ -224,5 +225,15 @@ def parse_uart_packet(body: bytes) -> dict | None:
"speed": pkt.speed,
}
if lora.version == 3:
pkt = dataflux.telemetry_common.telemetry_common.unpack_packet3(payload)
return {
**base,
"type": "packet3",
"start_time": pkt.start_time,
"duration": pkt.duration,
"count": pkt.count,
}
print("Unknown payload")
return None

99
src/dataflux/services/telemetry/__init__.py
View File

@@ -2,6 +2,7 @@
# Copyright (C) 2026 Association Exergie <association.exergie@gmail.com>
# SPDX-License-Identifier: GPL-3.0-or-later
from datetime import datetime
from queue import Empty
from dataflux.state import AppState, Buffers
import time
@@ -19,54 +20,59 @@ def telemetry_worker(state: AppState):
except Empty:
continue
state.latest_telemetry = dataframe
state.telemetry_valid = True
if dataframe["type"] == "packet2":
state.latest_telemetry = dataframe
state.telemetry_valid = True
with state.lock:
state.raw_buffers.timestamp.append(dataframe["time_stamp"])
state.raw_buffers.speed.append(dataframe["speed"])
state.raw_buffers.vbat.append(dataframe["vbat"])
state.raw_buffers.teng.append(dataframe["teng"])
state.raw_buffers.lat.append(dataframe["lat"])
state.raw_buffers.lng.append(dataframe["lng"])
with state.lock:
state.raw_buffers.timestamp.append(dataframe["time_stamp"])
state.raw_buffers.speed.append(dataframe["speed"])
state.raw_buffers.vbat.append(dataframe["vbat"])
state.raw_buffers.teng.append(dataframe["teng"])
state.raw_buffers.lat.append(dataframe["lat"])
state.raw_buffers.lng.append(dataframe["lng"])
state.live_buffers_updated = True
state.live_buffers.timestamp.clear()
state.live_buffers.speed.clear()
state.live_buffers.vbat.clear()
state.live_buffers.teng.clear()
state.live_buffers.lat.clear()
state.live_buffers.lng.clear()
state.live_buffers_updated = True
state.live_buffers.timestamp.clear()
state.live_buffers.speed.clear()
state.live_buffers.vbat.clear()
state.live_buffers.teng.clear()
state.live_buffers.lat.clear()
state.live_buffers.lng.clear()
if not state.raw_buffers.timestamp:
return
if not state.raw_buffers.timestamp:
return
last_timestamp = state.raw_buffers.timestamp[-1]
cutoff = last_timestamp - (state.live_buffer_len * 100)
last_timestamp = state.raw_buffers.timestamp[-1]
cutoff = last_timestamp - (state.live_buffer_len * 100)
i = len(state.raw_buffers.timestamp) - 1
i = len(state.raw_buffers.timestamp) - 1
while i >= 0 and state.raw_buffers.timestamp[i] >= cutoff:
elapsed_seconds = (
state.raw_buffers.timestamp[i] - last_timestamp
) / 100.0
state.live_buffers.timestamp.append(elapsed_seconds)
state.live_buffers.speed.append(state.raw_buffers.speed[i])
state.live_buffers.vbat.append(state.raw_buffers.vbat[i])
state.live_buffers.teng.append(state.raw_buffers.teng[i])
state.live_buffers.lat.append(state.raw_buffers.lat[i])
state.live_buffers.lng.append(state.raw_buffers.lng[i])
i -= 1
while i >= 0 and state.raw_buffers.timestamp[i] >= cutoff:
elapsed_seconds = (
state.raw_buffers.timestamp[i] - last_timestamp
) / 100.0
state.live_buffers.timestamp.append(elapsed_seconds)
state.live_buffers.speed.append(state.raw_buffers.speed[i])
state.live_buffers.vbat.append(state.raw_buffers.vbat[i])
state.live_buffers.teng.append(state.raw_buffers.teng[i])
state.live_buffers.lat.append(state.raw_buffers.lat[i])
state.live_buffers.lng.append(state.raw_buffers.lng[i])
i -= 1
state.live_buffers.timestamp.reverse()
state.live_buffers.speed.reverse()
state.live_buffers.vbat.reverse()
state.live_buffers.teng.reverse()
state.live_buffers.lat.reverse()
state.live_buffers.lng.reverse()
state.live_buffers.timestamp.reverse()
state.live_buffers.speed.reverse()
state.live_buffers.vbat.reverse()
state.live_buffers.teng.reverse()
state.live_buffers.lat.reverse()
state.live_buffers.lng.reverse()
elif dataframe["type"] == "packet3":
print(dataframe["type"])
print(dataframe["start_time"])
print(dataframe["duration"])
print(dataframe["count"])
def buffer_dump(state: AppState, path: str):
def buffer_dump(state: AppState, path: str) -> None:
save_path = Path(path)
if save_path.is_dir():
save_path = save_path / "output.csv"
@@ -97,3 +103,16 @@ def buffer_dump(state: AppState, path: str):
writer.writerow(row)
state.buffer_dump_thread = None
def autosave_worker(state: AppState, path: str) -> None:
output_dir = Path(path)
ctr: int = 0
while state.autosave_enabled:
date_str = state.start_time.strftime("%m_%d_%Y_%H_%M")
filename = date_str + ".csv"
save_path = output_dir / filename
buffer_dump(state, save_path)
print(f"Autosave {ctr} complete")
ctr += 1
time.sleep(30)

10
src/dataflux/state.py
View File

@@ -3,10 +3,13 @@
# SPDX-License-Identifier: GPL-3.0-or-later
from dataclasses import dataclass, field
from datetime import datetime
from threading import Lock, Thread
from serial import Serial
from queue import Queue
from serial.tools.list_ports_common import ListPortInfo
@dataclass
class Buffers:
@@ -21,6 +24,11 @@ class Buffers:
@dataclass
class AppState:
running: bool = True
start_time: datetime = datetime.now()
ports: list[ListPortInfo] = field(default_factory=list)
ports_thread: Thread | None = None
ports_thread_running: bool = False
lora_port: Serial | None = None
lora_thread: Thread | None = None
@@ -53,5 +61,7 @@ class AppState:
live_buffer_len: int = 30
buffer_dump_thread: Thread | None = None
autosave_buffer_thread: Thread | None = None
autosave_enabled: bool = False
lock: Lock = field(default_factory=Lock)

2
src/dataflux/tags.py
View File

@@ -7,11 +7,13 @@ MENU_IO_CONNECT_SERIAL: str = "menu_io_connect_serial"
MENU_IO_DISCONNECT_LORA: str = "menu_io_disconnect_lora"
MENU_IO_DISCONNECT_SERIAL: str = "menu_io_disconnect_serial"
MENU_FILE_DUMP_BUFFERS: str = "menu_file_dump_buffers"
MENU_FILE_AUTOSAVE_BUFFERS: str = "menu_file_autosave_buffers"
WINDOW_LORA_CONNECTION_MENU: str = "window_lora_connection_menu"
WINDOW_SERIAL_CONNECTION_MENU: str = "window_serial_connection_menu"
WINDOW_LORA_CONNECTION_MENU_COMBO: str = "window_lora_connection_menu_combo"
WINDOW_SERIAL_CONNECTION_MENU_COMBO: str = "window_serial_connection_menu_combo"
WINDOW_FILE_DIALOG_DUMP_BUFFERS: str = "window_file_dialog_dump_buffers"
WINDOW_FILE_DIALOG_AUTOSAVE_BUFFERS: str = "window_file_dialog_autosave_buffers"
CHILD_WINDOW_SERIAL_CONSOLE: str = "child_window_serial_console"

10
src/dataflux/ui/colors.py
View File

@@ -2,11 +2,11 @@
# Copyright (C) 2026 Association Exergie <association.exergie@gmail.com>
# SPDX-License-Identifier: GPL-3.0-or-later
STATUS_RED_DARK = (140, 35, 35, 255)
STATUS_RED_BRIGHT = (205, 85, 85, 255)
STATUS_RED_DARK = (140, 35, 35, 255)
STATUS_RED_BRIGHT = (205, 85, 85, 255)
STATUS_ORANGE_DARK = (160, 90, 20, 255)
STATUS_ORANGE_DARK = (160, 90, 20, 255)
STATUS_ORANGE_BRIGHT = (210, 140, 60, 255)
STATUS_GREEN_DARK = (40, 130, 55, 255)
STATUS_GREEN_BRIGHT = (95, 185, 115, 255)
STATUS_GREEN_DARK = (40, 130, 55, 255)
STATUS_GREEN_BRIGHT = (95, 185, 115, 255)

5
src/dataflux/ui/routines/windows.py
View File

@@ -3,7 +3,6 @@
# SPDX-License-Identifier: GPL-3.0-or-later
import dearpygui.dearpygui as dpg
import dataflux.config
from dataflux.services.serial import list_serial_ports
from dataflux.state import AppState
from dataflux.tags import (
@@ -18,7 +17,7 @@ from dataflux.tags import (
def update_window_lora_connection_menu_combo(state: AppState) -> None:
ports: list[str] = list_serial_ports()
ports: list[str] = list_serial_ports(state)
if state.serial_port is not None and state.serial_thread_running:
port_name = state.serial_port.name
@@ -28,7 +27,7 @@ def update_window_lora_connection_menu_combo(state: AppState) -> None:
def update_window_serial_connection_menu_combo(state: AppState) -> None:
ports: list[str] = list_serial_ports()
ports: list[str] = list_serial_ports(state)
if state.lora_port is not None and state.lora_thread_running:
port_name = state.lora_port.name

27
src/dataflux/ui/windows.py
View File

@@ -25,6 +25,7 @@ from dataflux.tags import (
LIVE_DATA_VBAT_VALUE,
LIVE_DATA_VEHICLE_TIME_VALUE,
LIVE_DATA_SPEED_VALUE,
MENU_FILE_AUTOSAVE_BUFFERS,
MENU_IO_CONNECT_LORA,
MENU_IO_DISCONNECT_LORA,
MENU_FILE_DUMP_BUFFERS,
@@ -43,6 +44,7 @@ from dataflux.tags import (
THEME_STATUS_CONNECTED,
THEME_STATUS_CONNECTED_BRIGHT,
THEME_STATUS_DISCONNECTED,
WINDOW_FILE_DIALOG_AUTOSAVE_BUFFERS,
WINDOW_LORA_CONNECTION_MENU,
WINDOW_LORA_CONNECTION_MENU_COMBO,
WINDOW_FILE_DIALOG_DUMP_BUFFERS,
@@ -74,7 +76,18 @@ def build_windows(state: AppState) -> None:
tag=MENU_FILE_DUMP_BUFFERS,
callback=dataflux.callbacks.menu.menu_file_dump_buffers,
)
dpg.add_menu_item(label="Quit")
dpg.add_menu_item(
label="Autosave Buffers",
enabled=True,
check=True,
default_value=False,
tag=MENU_FILE_AUTOSAVE_BUFFERS,
callback=dataflux.callbacks.menu.menu_file_autosave_buffers,
user_data=state,
)
dpg.add_menu_item(
label="Quit", callback=dataflux.callbacks.menu.menu_file_quit
)
with dpg.menu(label="IO"):
dpg.add_menu_item(
label="Connect LoRa",
@@ -427,3 +440,15 @@ def build_windows(state: AppState) -> None:
user_data=state,
):
dpg.add_file_extension(".csv")
with dpg.file_dialog(
directory_selector=True,
show=False,
tag=WINDOW_FILE_DIALOG_AUTOSAVE_BUFFERS,
width=700,
height=400,
modal=True,
callback=dataflux.callbacks.menu.window_file_dialog_autosave_buffers_ok,
user_data=state,
):
pass

12
src/dataflux/ui/worker.py
View File

@@ -7,13 +7,13 @@ import dearpygui.dearpygui as dpg
import datetime
import time
from datetime import datetime, timezone
import serial.tools.list_ports
from dataflux.state import AppState
from dataflux.tags import (
GRAPH_SERIES_SPEED,
GRAPH_SERIES_TENG,
GRAPH_SERIES_VBAT,
GRAPH_X_AXIS_SPEED,
LIVE_DATA_TENG_VALUE,
LIVE_DATA_UTC_TIME_VALUE,
LIVE_DATA_VBAT_VALUE,
@@ -23,6 +23,12 @@ from dataflux.tags import (
from dataflux.ui.routines.serial import append_text_to_console
def ports_worker(state: AppState) -> None:
while state.ports_thread_running:
state.ports = serial.tools.list_ports.comports()
time.sleep(5)
def ui_worker(state: AppState):
last_datetime: str = ""
last_veh_time: str = ""
@@ -31,6 +37,10 @@ def ui_worker(state: AppState):
last_teng: str = ""
no_data_written = False
while state.running:
# if state.autosave_enabled:
# dpg.set_value(MENU_FILE_AUTOSAVE_BUFFERS, True)
# else:
# dpg.set_value(MENU_FILE_AUTOSAVE_BUFFERS, False)
now = datetime.now(timezone.utc)
formatted = now.strftime("%H:%M:%S")