From 47773df57281b53351b8804b585c2dc4721a5f3d Mon Sep 17 00:00:00 2001
From: Hector van der Aa <hector@h3cx.dev>
Date: Fri, 18 Sep 2026 22:49:16 +0200
Subject: [PATCH] Added offload buffer intermediate and bugfix JSON structure
 leak

The core architecture has been updated, once recording is finished the
recording is automatically packed and loaded into an offload buffer,
leaving the processing buffer empty, loading the processing buffer is
now an explicit API call where passing None loads it from the offload
buffer and passing a Path loads it from an external .dlpak file

This means that the new recording -> playback architecture is as
follows:
recording -(automatic)-> offload -> processing -> playback

Loading offload (or an external .dlpak) into processing now requires the
core to be in the new 'offline' mode, and loading processing (or an
external .dlpak) into playback still requires the core to be in
'playback' mode
---
 src/dynalab_core/__init__.py | 36 ++++++++++++++++++++++++++++++++----
 src/dynalab_core/derive.py   |  9 +++++++--
 2 files changed, 39 insertions(+), 6 deletions(-)

diff --git a/src/dynalab_core/__init__.py b/src/dynalab_core/__init__.py
index 0b4c8c8..b6d389f 100644
--- a/src/dynalab_core/__init__.py
+++ b/src/dynalab_core/__init__.py
@@ -48,6 +48,7 @@ class Core:
         self._recording = threading.Event()
         self._recording_buffer = ValueBuffer()
         self._recording_timestamp: datetime = datetime.now(timezone.utc)
+        self._offload_buffer: DLPak | None = None
         self._processing_buffer: DLPak | None = None
         self._core_version: VersionDescriptor = CORE_VERSION
         self._stop_event: threading.Event = threading.Event()
@@ -248,7 +249,7 @@ class Core:
         if input is None:
             if self._processing_buffer is None:
                 raise CoreError(
-                    "Cannot load playback because no recording is available",
+                    "Cannot load playback because no processing buffer is available",
                     code="no_playback_data",
                 )
             loaded_data = copy(self._processing_buffer)
@@ -257,7 +258,33 @@ class Core:
         self._playback_engine.set_data(loaded_data)
 
     def load_processing_buffer(self, input: Path | None = None) -> None:
-        pass
+        if self._mode.value != "offline":
+            raise CoreError(
+                "Cannot load processing unless the core is in 'offline' mode",
+                code="invalid_mode",
+            )
+
+        if input is None:
+            if self._offload_buffer is None:
+                raise CoreError(
+                    "Cannot load processing because no recording buffer is available",
+                    code="no_playback_data",
+                )
+            loaded_data = DLPak().set_data(copy(self._recording_buffer))
+            connector_signal_descriptors = (
+                self._connector_registry.get_all_signal_descriptors()
+            )
+            derive_signal_descriptors = (
+                self._derive_registry.get_all_signal_descriptors()
+            )
+
+            loaded_data.set_manifest(
+                self._recording_timestamp,
+                connector_signal_descriptors + derive_signal_descriptors,
+            )
+        else:
+            loaded_data = DLPak().read(input)
+        self._processing_buffer = loaded_data
 
     # def unload_playback_engine
 
@@ -298,13 +325,14 @@ class Core:
         time.sleep(1)
         self._recording_buffer.normalize()
         self._recording_buffer.sort()
-        self._processing_buffer = DLPak().set_data(copy(self._recording_buffer))
+        self._offload_buffer = DLPak()
+        self._offload_buffer.set_data(copy(self._recording_buffer))
         connector_signal_descriptors = (
             self._connector_registry.get_all_signal_descriptors()
         )
         derive_signal_descriptors = self._derive_registry.get_all_signal_descriptors()
 
-        self._processing_buffer.set_manifest(
+        self._offload_buffer.set_manifest(
             self._recording_timestamp,
             connector_signal_descriptors + derive_signal_descriptors,
         )
diff --git a/src/dynalab_core/derive.py b/src/dynalab_core/derive.py
index 829d5f0..769b03d 100644
--- a/src/dynalab_core/derive.py
+++ b/src/dynalab_core/derive.py
@@ -255,11 +255,16 @@ class DeriveRegistry:
                 code="invalid_source",
             )
 
+        pure_input_signals = [
+            signal.to_pure() for signal in dlunit._manifest.input_signals
+        ]
+        pure_return_signal = dlunit._manifest.return_signal.to_pure()
+
         unit = DeriveUnit(
             unit_id,
             functions[0],
-            dlunit._manifest.input_signals,
-            dlunit._manifest.return_signal,
+            pure_input_signals,
+            pure_return_signal,
             self._core_input_queue,
         )