Exergie/dynalab-core
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added routing to core queue from endpoint

Browse Source
This commit is contained in:
h3cx
2026-08-05 16:38:15 +01:00
parent 6ec3ad6577
commit a882b29654
7 changed files with 463 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/dynalab_core/__init__.py
+33 -1
View File
@@ -3,7 +3,10 @@
# SPDX-License-Identifier: GPL-3.0-or-later
import logging
from queue import Empty, Queue
import threading
from threading import Thread
from time import monotonic, monotonic_ns, sleep
from typing import Literal
@@ -13,6 +16,8 @@ from dynalab_core.errors import CoreStateMismatchError
from dynalab_core.protocols.endpoint import ConnectorRegistry
from dynalab_core.protocols.common import VersionDescriptor
from dynalab_core.protocols.json.server import JsonServer
from dynalab_core.protocols.packets import ProtocolMessage
from dynalab_core.protocols.packets.data import ValueDescriptor
log = logging.getLogger(__name__)
@@ -27,7 +32,14 @@ class Core:
self._core_version: VersionDescriptor = CORE_VERSION
self._stop_event: threading.Event = threading.Event()
self._core_config: CoreConfig = config
self._connector_registry = ConnectorRegistry()
self._data_input_queue: Queue[ProtocolMessage] = Queue(524288)
self._input_worker_thread = Thread(
target=self._input_worker, name="input_worker_thread", daemon=True
)
self._input_worker_thread.start()
self._connector_registry = ConnectorRegistry(self._data_input_queue)
self._json_server = JsonServer(self._core_config, self._connector_registry)
self._state = "initd"
@@ -84,3 +96,23 @@ class Core:
"Core stopped",
extra={"event": "core.stopped", "core_state": self._state},
)
def _input_worker(self) -> None:
last_log = 0
while not self._stop_event.is_set():
now = monotonic_ns()
messages: list[ProtocolMessage] = []
try:
while not self._data_input_queue.empty():
messages.append(self._data_input_queue.get_nowait())
except Empty:
self._stop_event.wait(0.01)
continue
else:
for message in messages:
if isinstance(message, ValueDescriptor):
pass
if now > last_log + 2000 * 1_000_000:
log.debug(f"Core queue size: {self._data_input_queue.qsize()}")
last_log = now
src/dynalab_core/protocols/endpoint.py
+75 -45
View File
@@ -15,6 +15,7 @@ from dynalab_core.protocols.errors import (
ConnectorRegistryAlreadyRegisteredError,
)
from dynalab_core.protocols.packets import ProtocolMessage
from dynalab_core.protocols.packets.data import ValueDescriptor
from dynalab_core.protocols.packets.handshake import (
ConnectorHello,
HandshakeAccepted,
@@ -29,17 +30,21 @@ class ConnectorEndpoint:
def __init__(
self,
hello: ConnectorHello,
timeout_event: threading.Event | None = None,
timeout_event: threading.Event,
reject_event: threading.Event,
core_input_queue: Queue,
) -> None:
# external IO queues
self._packet_ingress_queue: Queue[ProtocolMessage] = Queue(524288)
self._packet_egress_queue: Queue[ProtocolMessage] = Queue(524288)
self._core_input_queue: Queue[ProtocolMessage] = core_input_queue
# internal IO queues
self._heartbeat_ingress_queue: Queue[ProtocolMessage] = Queue(524288)
self._heartbeat_egress_queue: Queue[ProtocolMessage] = Queue(524288)
self._connector_hello = hello
self._timed_out_event = timeout_event or threading.Event()
self._timed_out_event = timeout_event
self._reject_event = reject_event
self._stop_event = threading.Event()
self._input_worker_stopped_event = threading.Event()
self._input_worker_thread = Thread(
@@ -212,6 +217,11 @@ class ConnectorEndpoint:
else:
if isinstance(message, Heartbeat):
self._heartbeat_ingress_queue.put_nowait(message)
elif isinstance(message, ValueDescriptor):
if message.signal_id in (
signal.id for signal in self._connector_hello.signals
):
self._core_input_queue.put_nowait(message)
self._input_worker_stopped_event.set()
log.debug(
"Connector endpoint input worker %s stopped",
@@ -224,61 +234,81 @@ class ConnectorEndpoint:
)
def _output_worker(self) -> None:
connector_uuid = str(self.uuid())
log.debug(
"Connector endpoint output worker %s started",
self._input_worker_thread.name,
extra={
"event": "endpoint.output_worker_started",
"connector_uuid": connector_uuid,
"thread_name": self._input_worker_thread.name,
},
)
try:
connector_uuid = str(self.uuid())
log.debug(
"Connector endpoint output worker %s started",
self._input_worker_thread.name,
extra={
"event": "endpoint.output_worker_started",
"connector_uuid": connector_uuid,
"thread_name": self._input_worker_thread.name,
},
)
if (
not self._connector_hello.protocol_version
== HELLO_PACKET.protocol_version
):
self._put_egress_packet(
HandshakeRejected(reason="Protocol versions mismatch")
)
log.debug(f"Rejected handshake for endpoint {self.uuid()}")
self._reject_event.set()
return
if not self._connector_hello.signals:
self._put_egress_packet(
HandshakeRejected(reason="No signals available")
)
log.debug(f"Rejected handshake for endpoint {self.uuid()}")
self._reject_event.set()
return
if self._connector_hello.protocol_version == HELLO_PACKET.protocol_version:
self._put_egress_packet(HandshakeAccepted(accepted_signals=[]))
log.debug(f"Accepted handshake for endpoint {self.uuid()}")
else:
self._put_egress_packet(
HandshakeRejected(reason="Protocol versions mismatch")
log.debug(f"Accepted signals: {self._connector_hello.signals}")
self._heartbeat_worker_thread.start()
while not self._stop_event.is_set():
try:
message = self._heartbeat_egress_queue.get_nowait()
except Empty:
self._stop_event.wait(0.01)
pass
else:
self._put_egress_packet(message)
finally:
self._output_worker_stopped_event.set()
log.debug(
"Connector endpoint output worker %s stopped",
self._input_worker_thread.name,
extra={
"event": "endpoint.output_worker_stopped",
"connector_uuid": connector_uuid,
"thread_name": self._input_worker_thread.name,
},
)
log.debug(f"Rejected handshake for endpoint {self.uuid()}")
self._heartbeat_worker_thread.start()
while not self._stop_event.is_set():
messages: list[ProtocolMessage] = []
try:
messages.append(self._heartbeat_egress_queue.get_nowait())
except Empty:
self._stop_event.wait(0.01)
pass
for message in messages:
self._put_egress_packet(message)
self._output_worker_stopped_event.set()
log.debug(
"Connector endpoint output worker %s stopped",
self._input_worker_thread.name,
extra={
"event": "endpoint.output_worker_stopped",
"connector_uuid": connector_uuid,
"thread_name": self._input_worker_thread.name,
},
)
class ConnectorRegistry:
def __init__(self) -> None:
def __init__(self, core_input_queue: Queue) -> None:
self._endpoints: dict[UUID, ConnectorEndpoint] = {}
self._lock = RLock()
self._core_input_queue = core_input_queue
def register(
self, hello: ConnectorHello, timeout_event: threading.Event
self,
hello: ConnectorHello,
timeout_event: threading.Event,
reject_event: threading.Event,
) -> ConnectorEndpoint:
endpoint = ConnectorEndpoint(hello, timeout_event)
endpoint = ConnectorEndpoint(
hello, timeout_event, reject_event, self._core_input_queue
)
with self._lock:
current = self._endpoints.get(hello.connector_uuid)
src/dynalab_core/protocols/json/server.py
+6 -1
View File
@@ -272,8 +272,9 @@ class JsonServer:
connector_hello = message
timeout_event = threading.Event()
reject_event = threading.Event()
connector_endpoint = self._connector_registry.register(
connector_hello, timeout_event
connector_hello, timeout_event, reject_event
)
reason = "handler_completed"
@@ -302,11 +303,13 @@ class JsonServer:
timeout_task = asyncio.create_task(
self._wait_for_thread_event(timeout_event)
)
reject_task = asyncio.create_task(self._wait_for_thread_event(reject_event))
connection_tasks = [
input_task,
output_task,
server_stop_task,
timeout_task,
reject_task,
]
done, _ = await asyncio.wait(
@@ -314,6 +317,8 @@ class JsonServer:
)
if timeout_task in done:
reason = "endpoint_timeout"
elif reject_event in done:
reason = "connection_rejected"
elif server_stop_task in done:
reason = "server_shutdown"
src/dynalab_core/protocols/packets/__init__.py
+11 -1
View File
@@ -1,7 +1,12 @@
# Copyright (C) 2026 Hector van der Aa <hector@h3cx.dev>
# Copyright (C) 2026 Association Exergie <association.exergie@gmail.com>
# SPDX-License-Identifier: GPL-3.0-or-later
from typing import Annotated
from pydantic import Field
from dynalab_core.protocols.packets.data import ValueDescriptor
from dynalab_core.protocols.packets.handshake import (
ConnectorHello,
DynaLabHello,
@@ -12,6 +17,11 @@ from dynalab_core.protocols.packets.heartbeat import Heartbeat
ProtocolMessage = Annotated[
DynaLabHello | ConnectorHello | HandshakeAccepted | HandshakeRejected | Heartbeat,
DynaLabHello
| ConnectorHello
| HandshakeAccepted
| HandshakeRejected
| Heartbeat
| ValueDescriptor,
Field(discriminator="type"),
]
src/dynalab_core/protocols/packets/data.py
+13
View File
@@ -0,0 +1,13 @@
# Copyright (C) 2026 Hector van der Aa <hector@h3cx.dev>
# Copyright (C) 2026 Association Exergie <association.exergie@gmail.com>
# SPDX-License-Identifier: GPL-3.0-or-later
from typing import Literal
from uuid import UUID
from pydantic import BaseModel
class ValueDescriptor(BaseModel):
type: Literal["value_descriptor"] = "value_descriptor"
signal_id: UUID
value: float
src/dynalab_core/protocols/packets/handshake.py
+10 -2
View File
@@ -10,6 +10,15 @@ from dynalab_core.protocols.common import VersionDescriptor
from dynalab_core.protocols.constants import PROTOCOL_VERSION
class SignalDescriptor(BaseModel):
id: UUID
name: str
type: Literal["number", "binary"]
min_value: float | None = None
max_value: float | None = None
unit: str | None = None
class DynaLabHello(BaseModel):
type: Literal["dynalab_hello"] = "dynalab_hello"
instance_id: UUID
@@ -28,8 +37,7 @@ class ConnectorHello(BaseModel):
connector_name: str
connector_version: str
# TODO: implement SignalDescriptor
# signals: list[SignalDescriptor]
signals: list[SignalDescriptor]
class HandshakeAccepted(BaseModel):