Exergie/DataFlux
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added multiple data graphs and stronger safety with threads

Browse Source
This commit is contained in:
Hector van der Aa
2026-04-07 17:53:38 +02:00
parent 95ba7376c5
commit 2d631f0669
8 changed files with 311 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
src/dataflux/app.py
View File

@@ -2,10 +2,13 @@
# Copyright (C) 2026 Association Exergie <association.exergie@gmail.com>
# SPDX-License-Identifier: GPL-3.0-or-later
from threading import Thread
import dearpygui.dearpygui as dpg
from dataflux.state import AppState
import dataflux.ui.windows
import dataflux.ui.worker
import dataflux.services.telemetry
def run() -> None:
state: AppState = AppState()
@@ -30,6 +33,13 @@ def run() -> None:
dpg.configure_item("main_window", pos=(0, 0), width=vp_w, height=vp_h)
dpg.set_primary_window("main_window", True)
state.ui_worker_thread = Thread(target=dataflux.ui.worker.ui_worker, args=(state,), daemon=True)
state.ui_worker_thread.start()
state.telemetry_thread_running = True
state.telemetry_thread = Thread(target=dataflux.services.telemetry.telemetry_worker, args=(state, ), daemon=True)
state.telemetry_thread.start()
dpg.start_dearpygui()
dpg.destroy_context()

12
src/dataflux/callbacks/menu.py
View File

@@ -1,13 +1,16 @@
# Copyright (C) 2026 Hector van der Aa <hector@h3cx.dev>
# Copyright (C) 2026 Association Exergie <association.exergie@gmail.com>
# SPDX-License-Identifier: GPL-3.0-or-later
from threading import Thread
import dearpygui.dearpygui as dpg
from dataflux.state import AppState
from dataflux.ui.routines import update_global_connection_status
import dataflux.ui.routines.windows
import dataflux.ui.routines.status
import dataflux.services.serial
import dataflux.services.telemetry
from dataflux.tags import WINDOW_CONNECTION_MENU
from dataflux.tags import WINDOW_CONNECTION_MENU, WINDOW_FILE_DIALOG_DUMP_BUFFERS
def open_connection_window(sender, app_data, user_data) -> None:
dataflux.ui.routines.windows.update_window_connection_menu_combo()
@@ -17,3 +20,10 @@ def menu_file_disconnect(sender, app_data, user_data) -> None:
dataflux.services.serial.disconnect_serial(user_data)
update_global_connection_status(user_data)
def menu_file_dump_buffers(sender, app_data, user_data: AppState) -> None:
dpg.show_item(WINDOW_FILE_DIALOG_DUMP_BUFFERS)
def window_file_dialog_dump_buffers_ok(sender, app_data, user_data: AppState) -> None:
user_data.buffer_dump_thread = Thread(target=dataflux.services.telemetry.buffer_dump, args=(user_data, app_data["file_path_name"]), daemon=True)
user_data.buffer_dump_thread.start()

9
src/dataflux/services/serial/__init__.py
View File

@@ -10,6 +10,7 @@ import serial.tools.list_ports
from dataflux import telemetry_common
import dataflux.telemetry_common.telemetry_common
import dataflux.ui.routines.status
import dataflux.ui.routines
from dataflux.state import AppState
@@ -58,6 +59,9 @@ def serial_reader_worker(state: AppState) -> None:
port = state.serial_port
if port is None:
break
if port.closed:
print("Port closed")
break
try:
packet = read_one_uart_packet(port)
@@ -68,10 +72,13 @@ def serial_reader_worker(state: AppState) -> None:
if parsed is not None:
state.packet_queue.put(parsed)
state.serial_status_queue.put(0.1)
print(parsed)
except Exception as e:
print(f"Serial parser error: {e}")
break
disconnect_serial(state)
dataflux.ui.routines.update_global_connection_status(state)
def read_one_uart_packet(port: Serial) -> bytes | None:
first = port.read(1)

103
src/dataflux/services/telemetry/__init__.py Normal file
View File

@@ -0,0 +1,103 @@
# Copyright (C) 2026 Hector van der Aa <hector@h3cx.dev>
# Copyright (C) 2026 Association Exergie <association.exergie@gmail.com>
# SPDX-License-Identifier: GPL-3.0-or-later
from queue import Empty
from threading import local
from dataflux.state import AppState, Buffers
import time
from pathlib import Path
import csv
def hhmmsscc_to_day_seconds(value: int) -> int:
hours = value // 1000000
minutes = (value // 10000) % 100
seconds = (value // 100) % 100
return (hours * 3600) + (minutes * 60) + seconds
def telemetry_worker(state: AppState):
while state.telemetry_thread_running:
if state.serial_thread_running == False:
time.sleep(1)
continue
try:
dataframe = state.packet_queue.get(timeout=0.1)
except Empty:
continue
state.latest_telemetry = dataframe
state.telemetry_valid = True
with state.lock:
state.raw_buffers.timestamp.append(hhmmsscc_to_day_seconds(dataframe["time_stamp"]))
state.raw_buffers.speed.append(dataframe["speed"])
state.raw_buffers.vbat.append(dataframe["vbat"])
state.raw_buffers.teng.append(dataframe["teng"])
state.raw_buffers.lat.append(dataframe["lat"])
state.raw_buffers.lng.append(dataframe["lng"])
state.live_buffers_updated = True
state.live_buffers.timestamp.clear()
state.live_buffers.speed.clear()
state.live_buffers.vbat.clear()
state.live_buffers.teng.clear()
state.live_buffers.lat.clear()
state.live_buffers.lng.clear()
if not state.raw_buffers.timestamp:
return
last_timestamp = state.raw_buffers.timestamp[-1]
cutoff = last_timestamp - 30
i = len(state.raw_buffers.timestamp) - 1
while i >= 0 and state.raw_buffers.timestamp[i] >= cutoff:
state.live_buffers.timestamp.append(state.raw_buffers.timestamp[i] - last_timestamp)
state.live_buffers.speed.append(state.raw_buffers.speed[i])
state.live_buffers.vbat.append(state.raw_buffers.vbat[i])
state.live_buffers.teng.append(state.raw_buffers.teng[i])
state.live_buffers.lat.append(state.raw_buffers.lat[i])
state.live_buffers.lng.append(state.raw_buffers.lng[i])
i -= 1
state.live_buffers.timestamp.reverse()
state.live_buffers.speed.reverse()
state.live_buffers.vbat.reverse()
state.live_buffers.teng.reverse()
state.live_buffers.lat.reverse()
state.live_buffers.lng.reverse()
def buffer_dump(state: AppState, path: str):
print(path)
save_path = Path(path)
if save_path.is_dir():
save_path = save_path / "output.csv"
with state.lock:
local_raw_buffers = Buffers(
timestamp=list(state.raw_buffers.timestamp),
speed=list(state.raw_buffers.speed),
vbat=list(state.raw_buffers.vbat),
teng=list(state.raw_buffers.teng),
lat=list(state.raw_buffers.lat),
lng=list(state.raw_buffers.lng),
)
print(local_raw_buffers.timestamp)
with save_path.open("w", newline="", encoding="utf-8") as f:
writer = csv.writer(f)
writer.writerow(["timestamp", "speed", "vbat", "teng", "lat", "lng"])
for row in zip(local_raw_buffers.timestamp, local_raw_buffers.speed, local_raw_buffers.vbat, local_raw_buffers.teng, local_raw_buffers.lat, local_raw_buffers.lng):
writer.writerow(row)
state.buffer_dump_thread = None

19
src/dataflux/state.py
View File

@@ -7,6 +7,15 @@ from threading import Lock, Thread
from serial import Serial
from queue import Queue
@dataclass
class Buffers:
timestamp: list[int] = field(default_factory=list)
speed: list[float] = field(default_factory=list)
vbat: list[float] = field(default_factory=list)
teng: list[float] = field(default_factory=list)
lat: list[float] = field(default_factory=list)
lng: list[float] = field(default_factory=list)
@dataclass
class AppState:
running: bool = True
@@ -16,11 +25,21 @@ class AppState:
serial_thread_running: bool = False
telemetry_thread: Thread | None = None
telemetry_thread_running: bool = False
serial_status_thread: Thread | None = None
serial_status_queue: Queue = field(default_factory=Queue)
ui_worker_thread: Thread | None = None
packet_queue: Queue = field(default_factory=Queue)
latest_telemetry: dict = field(default_factory=dict)
telemetry_valid: bool = False
raw_buffers: Buffers = field(default_factory=Buffers)
live_buffers: Buffers = field(default_factory=Buffers)
live_buffers_updated: bool = False
buffer_dump_thread: Thread | None = None
lock: Lock = field(default_factory=Lock)

20
src/dataflux/tags.py
View File

@@ -4,12 +4,32 @@
MENU_FILE_CONNECT: str = "menu_file_connect"
MENU_FILE_DISCONNECT: str = "menu_file_disconnect"
MENU_FILE_DUMP_BUFFERS: str = "menu_file_dump_buffers"
WINDOW_CONNECTION_MENU: str = "window_connection_menu"
WINDOW_CONNECTION_MENU_COMBO: str = "window_connection_menu_combo"
WINDOW_FILE_DIALOG_DUMP_BUFFERS: str = "window_file_dialog_dump_buffers"
STATUS_SERIAL_STATUS_BOX: str = "status_serial_status_box"
STATUS_SERIAL_STATUS_TEXT: str = "status_serial_status_text"
LIVE_DATA_UTC_TIME_VALUE: str = "live_data_utc_time_value"
LIVE_DATA_VEHICLE_TIME_VALUE: str = "live_data_vehicle_time_value"
LIVE_DATA_SPEED_VALUE: str = "live_data_speed_value"
LIVE_DATA_VBAT_VALUE: str = "live_data_vbat_value"
LIVE_DATA_TENG_VALUE: str = "live_data_teng_value"
THEME_STATUS_DISCONNECTED: str = "theme_status_disconnected"
THEME_STATUS_CONNECTED: str = "theme_status_connected"
THEME_STATUS_CONNECTED_BRIGHT: str = "theme_status_connected_bigght"
GRAPH_X_AXIS_SPEED: str = "graph_x_axis_speed"
GRAPH_Y_AXIS_SPEED: str = "graph_y_axis_speed"
GRAPH_SERIES_SPEED: str = "graph_series_speed"
GRAPH_X_AXIS_VBAT: str = "graph_x_axis_vbat"
GRAPH_Y_AXIS_VBAT: str = "graph_y_axis_vbat"
GRAPH_SERIES_VBAT: str = "graph_series_vbat"
GRAPH_X_AXIS_TENG: str = "graph_x_axis_teng"
GRAPH_Y_AXIS_TENG: str = "graph_y_axis_teng"
GRAPH_SERIES_TENG: str = "graph_series_teng"

59
src/dataflux/ui/windows.py
View File

@@ -7,9 +7,18 @@ import dataflux.callbacks.menu
import dataflux.callbacks.serial
from dataflux.state import AppState
from dataflux.tags import MENU_FILE_CONNECT, MENU_FILE_DISCONNECT, STATUS_SERIAL_STATUS_BOX, STATUS_SERIAL_STATUS_TEXT, THEME_STATUS_CONNECTED, THEME_STATUS_CONNECTED_BRIGHT, THEME_STATUS_DISCONNECTED, WINDOW_CONNECTION_MENU, WINDOW_CONNECTION_MENU_COMBO
from dataflux.tags import GRAPH_SERIES_SPEED, GRAPH_SERIES_TENG, GRAPH_SERIES_VBAT, GRAPH_X_AXIS_SPEED, GRAPH_X_AXIS_TENG, GRAPH_X_AXIS_VBAT, GRAPH_Y_AXIS_SPEED, GRAPH_Y_AXIS_TENG, GRAPH_Y_AXIS_VBAT, LIVE_DATA_TENG_VALUE, LIVE_DATA_UTC_TIME_VALUE, LIVE_DATA_VBAT_VALUE, LIVE_DATA_VEHICLE_TIME_VALUE, LIVE_DATA_SPEED_VALUE, MENU_FILE_CONNECT, MENU_FILE_DISCONNECT, MENU_FILE_DUMP_BUFFERS, STATUS_SERIAL_STATUS_BOX, STATUS_SERIAL_STATUS_TEXT, THEME_STATUS_CONNECTED, THEME_STATUS_CONNECTED_BRIGHT, THEME_STATUS_DISCONNECTED, WINDOW_CONNECTION_MENU, WINDOW_CONNECTION_MENU_COMBO, WINDOW_FILE_DIALOG_DUMP_BUFFERS
from dataflux.ui.colors import STATUS_GREEN_BRIGHT, STATUS_GREEN_DARK, STATUS_RED_DARK
def _add_live_data_row(label: str, value: str, tag: str, units: str) -> None:
with dpg.table_row():
with dpg.table_cell():
dpg.add_text(label)
with dpg.table_cell():
dpg.add_text(value, tag=tag)
with dpg.table_cell():
dpg.add_text(units)
def build_windows(state: AppState) -> None:
with dpg.window(label='DataFlux',tag="main_window", no_collapse=True):
@@ -17,6 +26,7 @@ def build_windows(state: AppState) -> None:
with dpg.menu(label='File'):
dpg.add_menu_item(label="Connect", enabled=True, tag=MENU_FILE_CONNECT, callback=dataflux.callbacks.menu.open_connection_window)
dpg.add_menu_item(label="Disonnect", enabled=False, tag=MENU_FILE_DISCONNECT, callback=dataflux.callbacks.menu.menu_file_disconnect, user_data=state)
dpg.add_menu_item(label="Dump Buffers", enabled=True, tag=MENU_FILE_DUMP_BUFFERS, callback=dataflux.callbacks.menu.menu_file_dump_buffers)
dpg.add_menu_item(label="Quit")
with dpg.menu(label='Window'):
dpg.add_menu_item(label="Live Data", user_data="page_live_data")
@@ -25,16 +35,39 @@ def build_windows(state: AppState) -> None:
with dpg.child_window(tag="content_area", autosize_x=True, height=-32, border=False):
with dpg.group(tag="page_live_data", show=True):
with dpg.group(horizontal=True):
with dpg.child_window(tag="realtime_stats", width=250, autosize_y=True, border=True):
dpg.add_text("Speed: 25kmh")
with dpg.child_window(tag="realtime_stats", width=260, autosize_y=True, border=True):
with dpg.table(header_row=False, resizable=False, policy=dpg.mvTable_SizingFixedFit, borders_innerH=False, borders_innerV=False, borders_outerH=False, borders_outerV=False, no_host_extendX=True):
dpg.add_table_column(width_fixed=True)
dpg.add_table_column(width_stretch=True, init_width_or_weight=1.0)
dpg.add_table_column(width_fixed=True)
_add_live_data_row("UTC Time", "no_data", LIVE_DATA_UTC_TIME_VALUE,"")
_add_live_data_row("Vehicle Time", "no_data", LIVE_DATA_VEHICLE_TIME_VALUE,"")
_add_live_data_row("Speed", "no_data", LIVE_DATA_SPEED_VALUE,"km/h")
_add_live_data_row("Battery Voltage", "no_data", LIVE_DATA_VBAT_VALUE,"V")
_add_live_data_row("Engine Temp", "no_data", LIVE_DATA_TENG_VALUE,"°C")
with dpg.child_window(tag="data_graphs", autosize_x=True, autosize_y=True, border=True):
with dpg.plot(label="Speed", height=250, width=-1, no_inputs=True):
dpg.add_plot_legend()
dpg.add_plot_axis(dpg.mvXAxis, label="Time", tag="x_axis_speed")
y_axis = dpg.add_plot_axis(dpg.mvYAxis, label="Speed", tag="y_axis_speed")
dpg.set_axis_limits("y_axis_speed", 0, 50)
dpg.add_line_series([], [], parent=y_axis, tag="speed_series")
dpg.add_plot_axis(dpg.mvXAxis, label="Time", tag=GRAPH_X_AXIS_SPEED)
y_axis_speed = dpg.add_plot_axis(dpg.mvYAxis, label="Speed", tag=GRAPH_Y_AXIS_SPEED)
dpg.set_axis_limits(GRAPH_Y_AXIS_SPEED, ymin=0, ymax=50)
dpg.set_axis_limits(GRAPH_X_AXIS_SPEED, ymin=-30, ymax=0)
dpg.add_line_series([], [], parent=y_axis_speed, tag=GRAPH_SERIES_SPEED)
with dpg.plot(label="Battery Voltage", height=250, width=-1, no_inputs=True):
dpg.add_plot_legend()
dpg.add_plot_axis(dpg.mvXAxis, label="Time", tag=GRAPH_X_AXIS_VBAT)
y_axis_vbat = dpg.add_plot_axis(dpg.mvYAxis, label="Voltage", tag=GRAPH_Y_AXIS_VBAT)
dpg.set_axis_limits(GRAPH_Y_AXIS_VBAT, ymin=0, ymax=20)
dpg.set_axis_limits(GRAPH_X_AXIS_VBAT, ymin=-30, ymax=0)
dpg.add_line_series([], [], parent=y_axis_vbat, tag=GRAPH_SERIES_VBAT)
with dpg.plot(label="Engine Temp", height=250, width=-1, no_inputs=True):
dpg.add_plot_legend()
dpg.add_plot_axis(dpg.mvXAxis, label="Time", tag=GRAPH_X_AXIS_TENG)
y_axis_teng = dpg.add_plot_axis(dpg.mvYAxis, label="Temperature", tag=GRAPH_Y_AXIS_TENG)
dpg.set_axis_limits(GRAPH_Y_AXIS_TENG, ymin=0, ymax=120)
dpg.set_axis_limits(GRAPH_X_AXIS_TENG, ymin=-30, ymax=0)
dpg.add_line_series([], [], parent=y_axis_teng, tag=GRAPH_SERIES_TENG)
with dpg.group(tag="page_lap_recap", show=False):
dpg.add_text("Lap Recap")
@@ -74,3 +107,15 @@ def build_windows(state: AppState) -> None:
with dpg.window(label="Connection Menu", tag=WINDOW_CONNECTION_MENU, show=False, modal=True, no_collapse=True, width=300):
dpg.add_combo([], tag=WINDOW_CONNECTION_MENU_COMBO)
dpg.add_button(label="Connect", callback=dataflux.callbacks.serial.connection_window_connect_serial, user_data=state)
with dpg.file_dialog(
directory_selector=False,
show=False,
tag=WINDOW_FILE_DIALOG_DUMP_BUFFERS,
width=700,
height=400,
modal=True,
callback=dataflux.callbacks.menu.window_file_dialog_dump_buffers_ok,
user_data=state
):
dpg.add_file_extension(".csv")

88
src/dataflux/ui/worker.py Normal file
View File

@@ -0,0 +1,88 @@
# Copyright (C) 2026 Hector van der Aa <hector@h3cx.dev>
# Copyright (C) 2026 Association Exergie <association.exergie@gmail.com>
# SPDX-License-Identifier: GPL-3.0-or-later
import dearpygui.dearpygui as dpg
import datetime
import time
from datetime import datetime, timezone
from dataflux.state import AppState
from dataflux.tags import GRAPH_SERIES_SPEED, GRAPH_SERIES_TENG, GRAPH_SERIES_VBAT, GRAPH_X_AXIS_SPEED, LIVE_DATA_TENG_VALUE, LIVE_DATA_UTC_TIME_VALUE, LIVE_DATA_VBAT_VALUE, LIVE_DATA_VEHICLE_TIME_VALUE, LIVE_DATA_SPEED_VALUE
def ui_worker(state: AppState):
last_datetime: str = ""
last_veh_time: str = ""
last_veh_speed: str = ""
last_vbat: str = ""
last_teng: str = ""
no_data_written = False
while state.running:
now = datetime.now(timezone.utc)
formatted = now.strftime("%H:%M:%S")
if formatted != last_datetime:
dpg.set_value(LIVE_DATA_UTC_TIME_VALUE, formatted)
last_datetime = formatted
if state.serial_thread_running and state.telemetry_valid:
with state.lock:
# Vehicle Time
no_data_written = False
veh_time = state.latest_telemetry["time_stamp"]
veh_speed = state.latest_telemetry["speed"]
vbat = state.latest_telemetry["vbat"]
teng = state.latest_telemetry["teng"]
x_common = state.live_buffers.timestamp
speed_y = state.live_buffers.speed
vbat_y = state.live_buffers.vbat
teng_y = state.live_buffers.teng
hours = veh_time // 1000000
minutes = (veh_time // 10000) % 100
seconds = (veh_time // 100) % 100
formatted = f"{hours:02d}:{minutes:02d}:{seconds:02d}"
if formatted != last_veh_time:
dpg.set_value(LIVE_DATA_VEHICLE_TIME_VALUE, formatted)
last_veh_time = formatted
# Speed
formatted = f"{veh_speed:05.2f}"
if formatted != last_veh_speed:
dpg.set_value(LIVE_DATA_SPEED_VALUE, formatted)
last_veh_speed = formatted
# VBAT
formatted = f"{vbat:05.2f}"
if formatted != last_vbat:
dpg.set_value(LIVE_DATA_VBAT_VALUE, formatted)
last_vbat = formatted
# TENG
formatted = f"{teng:05.2f}"
if formatted != last_teng:
dpg.set_value(LIVE_DATA_TENG_VALUE, formatted)
last_teng = formatted
if state.live_buffers_updated:
dpg.set_value(GRAPH_SERIES_SPEED, [x_common, speed_y])
dpg.set_value(GRAPH_SERIES_VBAT, [x_common, vbat_y])
dpg.set_value(GRAPH_SERIES_TENG, [x_common, teng_y])
with state.lock:
state.live_buffers_updated = False
else:
if not no_data_written:
dpg.set_value(LIVE_DATA_VEHICLE_TIME_VALUE, "no_data")
dpg.set_value(LIVE_DATA_SPEED_VALUE, "no_data")
dpg.set_value(LIVE_DATA_VBAT_VALUE, "no_data")
dpg.set_value(LIVE_DATA_TENG_VALUE, "no_data")
last_veh_time = last_veh_speed = last_vbat = last_teng = "no_data"
no_data_written = True
time.sleep(0.05)