Moved derive unit API to DLUnit

Derived units now use a DLUnit class similar to a DLPak to be easily
stored and loaded, the public API has been updated to either take a
DLUnit object or a Path where it reads the DLUnit from a *.dlunit file

The .dlunit file is a ZIP compressed archive containing two files, a
manifest.json which stores the units uuid, its input signals and its
output signal, and unit.py which is the python code for the unit, the
same rules apply (one python function with a specifically structured
signature) as when a string was passed in the old API

Errors have been added to have a DLUnitError similar to the DLPakError
as the DLUnit class is heavily inspired from the DLPak architecutre

Logging may still be primitive, this will be covered in the next AI
logging sweep to homogenize and clean up the logging aspect of things

test/manual/core.py

@@ -7,6 +7,7 @@ from rich.logging import RichHandler
 
 from dynalab_core import Core
 from dynalab_core.config import CoreConfig
+from dynalab_core.dlunit import DLUnit
 from dynalab_core.protocols.packets.handshake import SignalDescriptor
 
 logging.basicConfig(
@@ -29,9 +30,9 @@ log.info("Configured manual core on %s", config.bind_str())
 dl_core = Core(config)
 dl_core.start()
 
-
-unit_id = dl_core.bind_derive_unit(
-    processing_str,
+dl_unit = DLUnit()
+dl_unit.set_manifest(
+    uuid4(),
     [
         SignalDescriptor(
             id=signal_id_1, name="Dummy signal 1", type="number", timeout_ms=5000
@@ -42,6 +43,9 @@ unit_id = dl_core.bind_derive_unit(
     ],
     SignalDescriptor(id=uuid4(), name="Dummy sum", type="number", timeout_ms=5000),
 )
+dl_unit.set_data(processing_str)
+
+unit_id = dl_core.bind_derive_unit(dl_unit)
 
 # for i in range(10):
 #     dl_core.wait(1)

test/manual/core_tui.py

@@ -1,7 +1,9 @@
+from collections import deque
 import logging
 import select
 import sys
 import termios
+from threading import Lock
 import time
 import tty
 from contextlib import contextmanager
@@ -17,6 +19,7 @@ from rich.text import Text
 
 from dynalab_core import Core
 from dynalab_core.config import CoreConfig
+from dynalab_core.dlunit import DLUnit
 from dynalab_core.errors import DynaLabError
 from dynalab_core.protocols.packets.handshake import SignalDescriptor
 
@@ -24,6 +27,8 @@ from dynalab_core.protocols.packets.handshake import SignalDescriptor
 HOST = "127.0.0.1"
 PORT = 8765
 DISPLAY_FREQUENCY_HZ = 5.0
+LOG_HISTORY_SIZE = 1_000
+VISIBLE_LOG_LINES = 12
 
 signal_id_1: UUID = UUID("3f32cea3-d872-4c16-a0a2-54b57171aeb2")
 signal_id_2: UUID = UUID("6578ac37-99d1-410c-b3f7-d049339919b2")
@@ -45,10 +50,59 @@ def keyboard_input() -> Iterator[None]:
         termios.tcsetattr(file_descriptor, termios.TCSADRAIN, previous_settings)
 
 
+class TuiLogHandler(logging.Handler):
+    """Keep recent DynaLab Core log messages available to the live display."""
+
+    def __init__(self) -> None:
+        super().__init__()
+        self._entries: deque[tuple[str, str, str, str]] = deque(
+            maxlen=LOG_HISTORY_SIZE
+        )
+        self._lock = Lock()
+        self.setFormatter(
+            logging.Formatter(
+                "%(asctime)s %(levelname)-8s %(name)s: %(message)s",
+                datefmt="%H:%M:%S",
+            )
+        )
+
+    def emit(self, record: logging.LogRecord) -> None:
+        try:
+            message = record.getMessage().replace("\n", " ↳ ")
+            if record.exc_info is not None and record.exc_info[1] is not None:
+                message = f"{message} ({record.exc_info[1]})"
+            level = record.levelname
+            level_style = {
+                logging.DEBUG: "dim",
+                logging.INFO: "white",
+                logging.WARNING: "yellow",
+                logging.ERROR: "bold red",
+                logging.CRITICAL: "bold white on red",
+            }.get(record.levelno, "white")
+            with self._lock:
+                self._entries.append(
+                    (
+                        self.formatter.formatTime(record, datefmt="%H:%M:%S"),
+                        level,
+                        message,
+                        level_style,
+                    )
+                )
+        except Exception:
+            self.handleError(record)
+
+    def latest(self, limit: int) -> list[tuple[str, str, str, str]]:
+        with self._lock:
+            return list(self._entries)[-limit:]
+
+
 class Dashboard:
-    def __init__(self, core: Core, config: CoreConfig) -> None:
+    def __init__(
+        self, core: Core, config: CoreConfig, log_handler: TuiLogHandler
+    ) -> None:
         self.core = core
         self.config = config
+        self.log_handler = log_handler
         self.running = True
         self.playback_loaded = False
         self.message = "Waiting for a connector"
@@ -170,6 +224,22 @@ class Dashboard:
             "[bold]L[/bold] Load   [bold]Space[/bold] Play/Pause   "
             "[bold]Q[/bold] Quit"
         )
+        log_table = Table.grid(expand=True, padding=(0, 1))
+        log_table.add_column(style="dim", width=8, no_wrap=True)
+        log_table.add_column(width=8, no_wrap=True)
+        log_table.add_column(ratio=1, no_wrap=True, overflow="ellipsis")
+        log_entries = self.log_handler.latest(VISIBLE_LOG_LINES)
+        if log_entries:
+            for timestamp, level, message, style in log_entries:
+                log_table.add_row(
+                    Text(timestamp),
+                    Text(level, style=style),
+                    Text(message, style=style),
+                )
+        else:
+            log_table.add_row("", "", Text("No DynaLab Core logs yet", style="dim"))
+        for _ in range(VISIBLE_LOG_LINES - max(1, len(log_entries))):
+            log_table.add_row("", "", "")
 
         return Group(
             Panel(
@@ -181,6 +251,11 @@ class Dashboard:
             Panel(value_table, title="Live values", border_style="cyan"),
             Panel(Text(self.message, style=self.message_style), title="Status"),
             Panel(controls, title="Hotkeys", border_style="blue"),
+            Panel(
+                log_table,
+                title=f"Logs · latest {VISIBLE_LOG_LINES}",
+                border_style="magenta",
+            ),
         )
 
     def playback_status(self, playing: bool) -> str:
@@ -193,15 +268,23 @@ class Dashboard:
 
 def main() -> None:
     logging.basicConfig(level=logging.CRITICAL)
+    log_handler = TuiLogHandler()
+    dynalab_logger = logging.getLogger("dynalab_core")
+    previous_log_level = dynalab_logger.level
+    previous_propagation = dynalab_logger.propagate
+    dynalab_logger.setLevel(logging.DEBUG)
+    dynalab_logger.propagate = False
+    dynalab_logger.addHandler(log_handler)
     config = CoreConfig(host=HOST, port=PORT)
     core = Core(config)
     processing_source = Path(__file__).with_name("process.py").read_text(
         encoding="utf-8"
     )
-    dashboard = Dashboard(core, config)
+    dashboard = Dashboard(core, config, log_handler)
 
-    core.bind_derive_unit(
-        processing_source,
+    derive_unit = DLUnit()
+    derive_unit.set_manifest(
+        uuid4(),
         [
             SignalDescriptor(
                 id=signal_id_1,
@@ -224,13 +307,15 @@ def main() -> None:
             origin="derived",
         ),
     )
+    derive_unit.set_data(processing_source)
+    core.bind_derive_unit(dlunit=derive_unit, file=None)
 
     try:
         core.start()
         with keyboard_input(), Live(
             dashboard.render(),
             auto_refresh=False,
-            screen=True,
+            screen=False,
         ) as live:
             while dashboard.running:
                 update_started = time.monotonic()
@@ -245,6 +330,9 @@ def main() -> None:
         pass
     finally:
         core.stop()
+        dynalab_logger.removeHandler(log_handler)
+        dynalab_logger.setLevel(previous_log_level)
+        dynalab_logger.propagate = previous_propagation
 
 
 if __name__ == "__main__":

test/test_core.py

@@ -1,14 +1,16 @@
 import logging
+from pathlib import Path
 import threading
 from uuid import uuid4
 
 import pytest
 from dynalab_core import Core
 from dynalab_core.config import CoreConfig
+from dynalab_core.dlunit import DLUnit
 from dynalab_core.errors import CoreError
 from dynalab_core.protocols.endpoint import ConnectorEndpoint
 from dynalab_core.protocols.packets.data import ValueDescriptor
-from dynalab_core.protocols.packets.handshake import ConnectorHello
+from dynalab_core.protocols.packets.handshake import ConnectorHello, SignalDescriptor
 from test.common import find_available_port
 
 
@@ -99,3 +101,34 @@ def test_core_mode_is_shared_with_endpoints() -> None:
         assert raised.value.code == "no_playback_data"
     finally:
         core.stop()
+
+
+def test_core_binds_in_memory_dlunit() -> None:
+    core = Core(CoreConfig(port=find_available_port(8765)))
+    input_signal = SignalDescriptor(id=uuid4(), name="Input", type="number")
+    return_signal = SignalDescriptor(
+        id=uuid4(), name="Output", type="number", origin="derived"
+    )
+    dlunit = DLUnit()
+    dlunit.set_manifest(uuid4(), [input_signal], return_signal)
+    dlunit.set_data(
+        """
+from dynalab_core.protocols.packets.data import ValueDescriptor
+from dynalab_core.protocols.packets.handshake import SignalDescriptor
+
+def identity(value: ValueDescriptor, output: SignalDescriptor) -> ValueDescriptor:
+    return ValueDescriptor(signal_id=output.id, value=value.value, timestamp=value.timestamp)
+"""
+    )
+
+    try:
+        unit_id = core.bind_derive_unit(dlunit)
+        assert core.get_signal_descriptor(return_signal.id) == return_signal
+
+        with pytest.raises(CoreError, match="either a DLUnit or a file") as raised:
+            core.bind_derive_unit(dlunit, file=Path("unit.dlunit"))
+
+        assert raised.value.code == "bind_derive_failed"
+        core.unbind_derive_unit(unit_id)
+    finally:
+        core.stop()

test/test_derive.py

@@ -5,6 +5,7 @@ from uuid import UUID, uuid4
 
 import pytest
 from dynalab_core.derive import DeriveRegistry
+from dynalab_core.dlunit import DLUnit
 from dynalab_core.protocols.packets.data import ValueDescriptor
 from dynalab_core.protocols.packets.handshake import SignalDescriptor
 
@@ -30,6 +31,17 @@ def fail(value: ValueDescriptor, output: SignalDescriptor) -> ValueDescriptor:
 """
 
 
+def make_dlunit(
+    source: str,
+    input_signals: list[SignalDescriptor],
+    return_signal: SignalDescriptor,
+) -> DLUnit:
+    dlunit = DLUnit()
+    dlunit.set_manifest(uuid4(), input_signals, return_signal)
+    dlunit.set_data(source)
+    return dlunit
+
+
 def test_derive_registry_routes_with_live_values() -> None:
     output_queue: Queue[ValueDescriptor] = Queue()
     left_signal = SignalDescriptor(id=uuid4(), name="Left", type="number")
@@ -40,7 +52,11 @@ def test_derive_registry_routes_with_live_values() -> None:
 
     try:
         unit = registry.register(
-            PROCESSING_FUNCTION, [left_signal, right_signal], output_signal
+            make_dlunit(
+                PROCESSING_FUNCTION,
+                [left_signal, right_signal],
+                output_signal,
+            )
         )
         right_value = ValueDescriptor(signal_id=right_signal.id, value=2.0, timestamp=1)
         live_values[right_signal.id] = right_value
@@ -87,7 +103,11 @@ def test_derive_worker_logs_failure_and_stops(
     with caplog.at_level(logging.DEBUG, logger="dynalab_core"):
         try:
             unit = registry.register(
-                FAILING_PROCESSING_FUNCTION, [input_signal], output_signal
+                make_dlunit(
+                    FAILING_PROCESSING_FUNCTION,
+                    [input_signal],
+                    output_signal,
+                )
             )
             unit.put_data(
                 [ValueDescriptor(signal_id=input_signal.id, value=1.0, timestamp=1)]

test/test_errors.py

@@ -8,6 +8,7 @@ import pytest
 from dynalab_core.buffer import ValueBuffer
 from dynalab_core.derive import DeriveRegistry, DeriveUnit
 from dynalab_core.dlpak import DLPak
+from dynalab_core.dlunit import DLUnit
 from dynalab_core.errors import (
     CoreError,
     DLPakError,
@@ -97,7 +98,10 @@ def test_derive_registry_wraps_invalid_source() -> None:
 
     try:
         with pytest.raises(DeriveError, match="Could not load") as raised:
-            registry.register("def broken(", [], output_signal)
+            dlunit = DLUnit()
+            dlunit.set_manifest(uuid4(), [], output_signal)
+            dlunit.set_data("def broken(")
+            registry.register(dlunit)
     finally:
         registry.stop()