From 7f7ab5585be43c4fc79a07c71a600f65b9c39c4d Mon Sep 17 00:00:00 2001 From: lestarch-autobot Date: Thu, 29 Oct 2026 17:07:33 +0000 Subject: [PATCH 1/8] Bind `hook` queue-full overflow hooks to Python Model each overflow hook as a HandlerView so the existing generators emit the forwarding C++ override and the Python stub; drop the unsupported-feature rejection. Co-authored-by: M Starch --- src/fprime_python/autocode/view.py | 89 +++++++++++++++++++----------- 1 file changed, 57 insertions(+), 32 deletions(-) diff --git a/src/fprime_python/autocode/view.py b/src/fprime_python/autocode/view.py index a7ec6ce..8b0c5ce 100644 --- a/src/fprime_python/autocode/view.py +++ b/src/fprime_python/autocode/view.py @@ -312,6 +312,36 @@ def parameters(self) -> List[FormalParameterView]: ] +class OverflowHookView(HandlerView): + """ The overflow hook F Prime declares for a member with `hook` queue-full behavior + + F Prime calls it instead of the handler when the message cannot be queued. It takes the handler's arguments, + except that a command's hook gets only the opcode and the command sequence number. + """ + + def __init__(self, owner: HandlerView, suffix: str) -> None: + """ Wrap the hook of a port, internal port or command, F Prime naming it with the given suffix """ + self.owner = owner + self.suffix = suffix + self.HANDLER_PREFIX = owner.HANDLER_PREFIX + self.HANDLER_KIND = f"queue overflow of {owner.HANDLER_KIND}" + + @property + def name(self) -> str: + """ The owning member's name as written in the model """ + return self.owner.name + + @property + def handler_name(self) -> str: + """ The name of the overflow hook F Prime declares for the owning member """ + return f"{self.owner.name}_{self.suffix}" + + @property + def parameters(self) -> List[FormalParameterView]: + """ The owning member's formal parameters, none for a command """ + return [] if isinstance(self.owner, CommandView) else self.owner.parameters + + class EventView: """ One event of a component """ @@ -459,49 +489,43 @@ def has_commands(self) -> bool: return bool(self.commands) @property - def has_queue_full_hooks(self) -> bool: - """ Whether anything on the component declares `hook` queue-full behavior - - `hook` makes F Prime declare a pure virtual overflow hook -- `_overflowHook`, - `_overflowHook` or `_cmdOverflowHook` -- alongside the handler. Async input ports, - internal ports and async commands can each ask for one. + def overflow_hooks(self) -> List[OverflowHookView]: + """ The overflow hooks of the async input ports, internal ports and async commands with `hook` queue-full + behavior, in that order """ - for port in self.component.port_map.values(): - if isinstance(port, fpp.PortInstance.General): - kind = port.kind - if ( - isinstance(kind, fpp.GeneralKind.AsyncInput) - and kind.queue_full == fpp.ast.QueueFull.Hook - ): - return True - elif ( - isinstance(port, fpp.PortInstance.Internal) - and port.queue_full == fpp.ast.QueueFull.Hook - ): - return True - return any( - isinstance(command.command.kind, fpp.NonParamKind.Async) - and command.command.kind.queue_full == fpp.ast.QueueFull.Hook + hooks = [ + OverflowHookView(port, "overflowHook") + for port in self.input_ports + if isinstance(port.port.kind, fpp.GeneralKind.AsyncInput) + and port.port.kind.queue_full == fpp.ast.QueueFull.Hook + ] + hooks += [ + OverflowHookView(port, "overflowHook") + for port in self.internal_ports + if port.port.queue_full == fpp.ast.QueueFull.Hook + ] + hooks += [ + OverflowHookView(command, "cmdOverflowHook") for command in self.commands - ) + if isinstance(command.command.kind, fpp.NonParamKind.Async) + and command.command.kind.queue_full == fpp.ast.QueueFull.Hook + ] + return hooks def check_supported(self) -> None: """ Reject a component whose features this autocoder cannot bind - F Prime declares a pure virtual handler for every state machine action and guard, for every data - product container, and for every `hook` queue-full overflow. None has a Python counterpart yet -- - an action takes a state machine id and a signal, a container handler takes a `Fw::DpContainer`, and - an overflow hook takes the message that could not be queued -- so generating the component anyway - would leave an abstract class and fail at link time, a long way from the cause. + F Prime declares a pure virtual handler for every state machine action and guard, and for every data + product container. Neither has a Python counterpart yet -- an action takes a state machine id and a + signal, and a container handler takes a `Fw::DpContainer` -- so generating the component anyway would + leave an abstract class and fail at link time, a long way from the cause. Raises: - UnsupportedModelError: The component has state machine instances, data products or a `hook` - queue-full declaration + UnsupportedModelError: The component has state machine instances or data products """ unsupported = [ ("state machine instances", self.component.has_state_machine_instances), ("data products", self.component.has_data_products), - ("`hook` queue-full behavior", self.has_queue_full_hooks), ] for feature, present in unsupported: if present: @@ -576,12 +600,13 @@ def commands(self) -> List[CommandView]: def handlers(self) -> List[HandlerView]: """ Every component member F Prime declares a handler for, in generation order - The three families get the same forwarding C++ override and a stub in the Python template (a command's + Every family gets the same forwarding C++ override and a stub in the Python template (a command's stub also sends its response), so the generators walk this one list rather than each family in turn. """ handlers: List[HandlerView] = list(self.input_ports) handlers += self.internal_ports handlers += self.commands + handlers += self.overflow_hooks return handlers @property From 093736a2add6782408144d28df1c89815aa5803e Mon Sep 17 00:00:00 2001 From: lestarch-autobot Date: Thu, 29 Oct 2026 17:24:57 +0000 Subject: [PATCH 2/8] Answer BUSY from the command overflow hook stub F Prime drops an overflowed command unanswered, so the template's command hook sends BUSY; hook kinds read as ' overflow hook'. Co-authored-by: M Starch --- .../autocode/component_generator.py | 24 +++++++++++++------ src/fprime_python/autocode/view.py | 2 +- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/src/fprime_python/autocode/component_generator.py b/src/fprime_python/autocode/component_generator.py index 0661e1b..4bb8a52 100644 --- a/src/fprime_python/autocode/component_generator.py +++ b/src/fprime_python/autocode/component_generator.py @@ -18,7 +18,14 @@ from .binding_generator import BindingGenerator, expression_chain, standard_def from .constants import SELF_MEMBER, SUPPORT_HEADER, TOOL_NAME from .include import IncludeManager -from .view import CommandView, ComponentView, FormalParameterView, HandlerView, ParameterView +from .view import ( + CommandView, + ComponentView, + FormalParameterView, + HandlerView, + OverflowHookView, + ParameterView, +) #: Telemetry write bindings keep the default timestamp argument F Prime declares, so that Python callers @@ -277,11 +284,11 @@ class {name}({name}Base): pass ''' -#: Stub for one command handler in the Python implementation template +#: Stub for one command handler or command overflow hook in the Python implementation template PYTHON_COMMAND_HANDLER_TEMPLATE = ''' def {handler}(self{args}): - """ Handle the {name} command """ - # TODO: Implement command handler - self.cmdResponse_out({prefix}, fprime_py.Fw.CmdResponse(fprime_py.Fw.CmdResponse.T.OK)) + """ Handle the {name} {kind} """ + # TODO: Implement {kind} handler + self.cmdResponse_out({prefix}, fprime_py.Fw.CmdResponse(fprime_py.Fw.CmdResponse.T.{response})) ''' @@ -556,14 +563,17 @@ def python_implementation(self) -> str: for handler in self.component.handlers: args = "".join(f", {name}" for name in handler.handler_arguments) # A command handler is not left to fall off the end: the ground waits for a response, so the - # stub sends one - if isinstance(handler, CommandView): + # stub sends one. A command's overflow hook answers too, since F Prime dropped the command unanswered + overflowed = isinstance(handler, OverflowHookView) and isinstance(handler.owner, CommandView) + if isinstance(handler, CommandView) or overflowed: stubs.append( PYTHON_COMMAND_HANDLER_TEMPLATE.format( handler=handler.handler_name, name=handler.name, + kind=handler.HANDLER_KIND, args=args, prefix=", ".join(name for _, name, _ in handler.HANDLER_PREFIX), + response="BUSY" if overflowed else "OK", ) ) continue diff --git a/src/fprime_python/autocode/view.py b/src/fprime_python/autocode/view.py index 8b0c5ce..679e95f 100644 --- a/src/fprime_python/autocode/view.py +++ b/src/fprime_python/autocode/view.py @@ -324,7 +324,7 @@ def __init__(self, owner: HandlerView, suffix: str) -> None: self.owner = owner self.suffix = suffix self.HANDLER_PREFIX = owner.HANDLER_PREFIX - self.HANDLER_KIND = f"queue overflow of {owner.HANDLER_KIND}" + self.HANDLER_KIND = f"{owner.HANDLER_KIND} overflow hook" @property def name(self) -> str: From 8d5af97cd1f7f0caa6909716b7b645abe36877bb Mon Sep 17 00:00:00 2001 From: lestarch-autobot Date: Thu, 29 Oct 2026 17:24:57 +0000 Subject: [PATCH 3/8] Test overflow hook generation Hooks.fpp fixture covers async input port, internal port and async command hooks, and a drop port that gets none; Config.fpp gains command port stand-ins. Co-authored-by: M Starch --- tests/fixtures/Config.fpp | 8 +++++++- tests/fixtures/Hooks.fpp | 18 ++++++++++++++++++ tests/test_generator.py | 31 +++++++++++++++++++++++++++++++ 3 files changed, 56 insertions(+), 1 deletion(-) create mode 100644 tests/fixtures/Hooks.fpp diff --git a/tests/fixtures/Config.fpp b/tests/fixtures/Config.fpp index f9b5e25..0828040 100644 --- a/tests/fixtures/Config.fpp +++ b/tests/fixtures/Config.fpp @@ -6,15 +6,21 @@ type FwSizeType = U64 type FwEventIdType = U32 type FwChanIdType = U32 +type FwOpcodeType = U32 -@ Stand-ins for the framework ports that events and telemetry need +@ Stand-ins for the framework ports that commands, events and telemetry need module Fw { type Time type LogBuffer type TlmBuffer + type CmdArgBuffer + enum CmdResponse { OK } enum LogSeverity { ACTIVITY_LO } port Log($id: FwEventIdType, ref timeTag: Fw.Time, $severity: LogSeverity, ref args: LogBuffer) port LogText($id: FwEventIdType, ref timeTag: Fw.Time, $severity: LogSeverity, ref $text: string size 256) port Time(ref $time: Fw.Time) + port Cmd(opCode: FwOpcodeType, cmdSeq: U32, ref args: CmdArgBuffer) + port CmdReg(opCode: FwOpcodeType) + port CmdResponse(opCode: FwOpcodeType, cmdSeq: U32, response: CmdResponse) port Tlm($id: FwChanIdType, ref timeTag: Fw.Time, ref val: TlmBuffer) } diff --git a/tests/fixtures/Hooks.fpp b/tests/fixtures/Hooks.fpp new file mode 100644 index 0000000..125f32d --- /dev/null +++ b/tests/fixtures/Hooks.fpp @@ -0,0 +1,18 @@ +module Fixture { + + @ Port with a string argument + port Text(msg: string size 16) + + @ A component asking for an overflow hook on every member that can have one + @ fprime-python + queued component PyHooks { + async input port textIn: Text hook + async input port dropIn: Text drop + internal port poke(msg: string size 5) hook + async command DO(n: U32) hook + command recv port cmdIn + command reg port cmdRegOut + command resp port cmdResponseOut + } + +} diff --git a/tests/test_generator.py b/tests/test_generator.py index b0e961a..df3e0f1 100644 --- a/tests/test_generator.py +++ b/tests/test_generator.py @@ -120,3 +120,34 @@ def test_sized_string_return_is_checked_against_its_declared_size(files: Dict[st assert check in source assert source.index("Fw::String result = return_value.cast();") < source.index(check) assert source.index(check) < source.index("return result;") + + +@pytest.fixture +def hooks(tmp_path: Path) -> Dict[str, str]: + return generate("Hooks.fpp", tmp_path) + + +def test_overflow_hooks_are_forwarded_into_python(hooks: Dict[str, str]) -> None: + header = hooks["PyHooks.hpp"] + assert "void textIn_overflowHook(" in header + assert "void poke_overflowHook(const Fw::InternalInterfaceString& msg) override;" in header + assert "void DO_cmdOverflowHook(" in header + source = hooks["PyHooks.cpp"] + assert 'm_self.attr("textIn_overflowHook")(portNum, msg);' in source + assert 'm_self.attr("poke_overflowHook")(msg);' in source + # A command's hook gets the opcode and sequence number, not the command's arguments + assert 'm_self.attr("DO_cmdOverflowHook")(opCode, cmdSeq);' in source + + +def test_overflow_hooks_get_python_stubs(hooks: Dict[str, str]) -> None: + template = hooks["PyHooks.template.py"] + assert "def textIn_overflowHook(self, portNum, msg):" in template + assert "def poke_overflowHook(self, msg):" in template + hook = template[template.index("def DO_cmdOverflowHook(self, opCode, cmdSeq):"):] + assert "self.cmdResponse_out(opCode, cmdSeq, fprime_py.Fw.CmdResponse(fprime_py.Fw.CmdResponse.T.BUSY))" in hook + + +def test_members_without_hook_get_no_overflow_hook(files: Dict[str, str], hooks: Dict[str, str]) -> None: + assert "dropIn_overflowHook" not in hooks["PyHooks.cpp"] + assert "overflowHook" not in files["PyQueued.cpp"] + assert "overflowHook" not in files["PyQueued.template.py"] From ae858175eb8974faab393ee4abae5088e3419e87 Mon Sep 17 00:00:00 2001 From: lestarch-autobot Date: Thu, 29 Oct 2026 17:24:57 +0000 Subject: [PATCH 4/8] Document queue-full overflow hooks Co-authored-by: M Starch --- README.md | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index cc77282..965e35d 100644 --- a/README.md +++ b/README.md @@ -69,19 +69,40 @@ Once finished, the python bindings will be autocoded and included in the next bu as shown below). This will also produce a `.template.py` file in the component folder as a basic template for implementing components in python. -Every port, command, event, telemetry channel, parameter and internal port of the component is bound. Four -component features cannot be bound yet, because F Prime declares a handler for each that has no Python equivalent: +Every port, command, event, telemetry channel, parameter and internal port of the component is bound, as is the +overflow hook of each member with `hook` queue-full behavior (see below). Three component features cannot be bound +yet, because F Prime declares a handler for each that has no Python equivalent: | Feature | Why | What happens | | --- | --- | --- | | serial port | its handler takes a serialization buffer | rejected with an error naming the port | | state machine instance | its actions and guards take a state machine id and a signal | rejected with an error naming the component | | data product container | its handler takes a `Fw::DpContainer` | rejected with an error naming the component | -| `hook` queue-full behavior | its overflow hook takes the message that could not be queued | rejected with an error naming the component | In each case the autocoder refuses rather than generating a class that would leave the handler unimplemented and fail to link. Drop the `@ fprime-python` annotation to implement such a component in C++ instead. +### Queue-Full Overflow Hooks + +An async input port, internal port or async command declared with `hook` queue-full behavior gets an overflow hook +in Python next to its handler. When the component's queue is full, F Prime calls the hook instead of queuing the +message, and the message is dropped once the hook returns. + +| Member | Hook | Arguments | +| --- | --- | --- | +| `async input port textIn: Text hook` | `textIn_overflowHook` | `portNum` and the port's arguments | +| `internal port poke(msg: string size 5) hook` | `poke_overflowHook` | the internal port's arguments | +| `async command DO(n: U32) hook` | `DO_cmdOverflowHook` | `opCode` and `cmdSeq` only | + +The hook runs on the thread of the caller that found the queue full, not on the component's thread, so keep it short +and do not block in it. F Prime does not answer a command it dropped: the template's command hook sends `BUSY` so the +ground is not left waiting. + +```python +def DO_cmdOverflowHook(self, opCode, cmdSeq): + self.cmdResponse_out(opCode, cmdSeq, fprime_py.Fw.CmdResponse(fprime_py.Fw.CmdResponse.T.BUSY)) +``` + ## Setting a Deployment Up With Python Bindings The deployment topology that instantiates Python components must also carry the `@ fprime-python` annotation. Only an From 362b2162f60bfe3e24a60cda99a6e926439f7f14 Mon Sep 17 00:00:00 2001 From: lestarch-autobot Date: Thu, 29 Oct 2026 17:33:40 +0000 Subject: [PATCH 5/8] Report missing overflow hooks at init The generated init checks the Python object implements every overflow hook, so a missing one fails at startup instead of at the first overflow on the caller's thread. OverflowHookView derives its own name and generated comment. Co-authored-by: M Starch --- .../autocode/component_generator.py | 11 ++++- src/fprime_python/autocode/view.py | 44 +++++++++++++------ 2 files changed, 39 insertions(+), 16 deletions(-) diff --git a/src/fprime_python/autocode/component_generator.py b/src/fprime_python/autocode/component_generator.py index 4bb8a52..b567903 100644 --- a/src/fprime_python/autocode/component_generator.py +++ b/src/fprime_python/autocode/component_generator.py @@ -353,6 +353,13 @@ def _write_init(self, body: Body) -> None: 'Call auto-coded initialization function storing the C++ "this" object for Python' ) body.line(f'this->{SELF_MEMBER}.attr("_init_ac")(this);') + # A hook runs only when the queue fills, so a missing one is reported at startup, not first overflow + for hook in self.component.overflow_hooks: + with body.if_(f'!pybind11::hasattr(this->{SELF_MEMBER}, "{hook.handler_name}")'): + body.line( + f'throw pybind11::attribute_error("{self.name} does not implement overflow hook' + f' {hook.handler_name}");' + ) body.comment("Continue the standard initialization of F Prime") arguments = ", ".join(name for _, name, _ in self._init_params()) body.line(f"{self.component.base_class}::init({arguments});") @@ -443,7 +450,7 @@ def _write_handlers(self, cls: ClassBuilder) -> None: ret=handler.handler_return_type, params=handler.handler_parameters, override=True, - comment=f"Handler for {handler.HANDLER_KIND} {handler.name}", + comment=handler.description, ) self._write_forwarding_call(override.body, handler) @@ -564,7 +571,7 @@ def python_implementation(self) -> str: args = "".join(f", {name}" for name in handler.handler_arguments) # A command handler is not left to fall off the end: the ground waits for a response, so the # stub sends one. A command's overflow hook answers too, since F Prime dropped the command unanswered - overflowed = isinstance(handler, OverflowHookView) and isinstance(handler.owner, CommandView) + overflowed = isinstance(handler, OverflowHookView) and handler.is_command_hook if isinstance(handler, CommandView) or overflowed: stubs.append( PYTHON_COMMAND_HANDLER_TEMPLATE.format( diff --git a/src/fprime_python/autocode/view.py b/src/fprime_python/autocode/view.py index 679e95f..1b89b55 100644 --- a/src/fprime_python/autocode/view.py +++ b/src/fprime_python/autocode/view.py @@ -109,11 +109,11 @@ class UnsupportedModelError(Exception): class HandlerView(ABC): - """ One component member that F Prime declares a pure virtual handler for + """ One pure virtual handler F Prime declares for a component member A handler's signature is a fixed prefix -- the port number for a general port, the opcode and the - command sequence number for a command, nothing for an internal port -- followed by the member's own - formal parameters. The generated C++ override, the call it forwards into Python, and the stub in the + command sequence number for a command, nothing for an internal port -- followed by its formal parameters: + the member's own, or none for a command's overflow hook. The generated C++ override, the call it forwards into Python, and the stub in the Python template all have to agree on that signature, so it is spelled once here and each generator reads it back rather than repeating the prefix. """ @@ -134,6 +134,11 @@ def name(self) -> str: def handler_name(self) -> str: """ The name of the handler F Prime declares for this member """ + @property + def description(self) -> str: + """ What the handler is, as it reads in a generated comment """ + return f"Handler for {self.HANDLER_KIND} {self.name}" + @property @abstractmethod def parameters(self) -> List[FormalParameterView]: @@ -316,16 +321,26 @@ class OverflowHookView(HandlerView): """ The overflow hook F Prime declares for a member with `hook` queue-full behavior F Prime calls it instead of the handler when the message cannot be queued. It takes the handler's arguments, - except that a command's hook gets only the opcode and the command sequence number. + except that a command's hook gets only the opcode and the command sequence number. The handler prefix is the + owner's, so it is copied onto the instance rather than declared by the class. """ - def __init__(self, owner: HandlerView, suffix: str) -> None: - """ Wrap the hook of a port, internal port or command, F Prime naming it with the given suffix """ + def __init__(self, owner: HandlerView) -> None: + """ Wrap the hook of a port, internal port or command """ self.owner = owner - self.suffix = suffix self.HANDLER_PREFIX = owner.HANDLER_PREFIX self.HANDLER_KIND = f"{owner.HANDLER_KIND} overflow hook" + @property + def is_command_hook(self) -> bool: + """ Whether the hook is a command's, which takes no command arguments and must answer the command """ + return isinstance(self.owner, CommandView) + + @property + def description(self) -> str: + """ The hook as F Prime's own generated comment names it """ + return f"Overflow hook for {self.owner.HANDLER_KIND} {self.name}" + @property def name(self) -> str: """ The owning member's name as written in the model """ @@ -334,12 +349,13 @@ def name(self) -> str: @property def handler_name(self) -> str: """ The name of the overflow hook F Prime declares for the owning member """ - return f"{self.owner.name}_{self.suffix}" + suffix = "cmdOverflowHook" if self.is_command_hook else "overflowHook" + return f"{self.owner.name}_{suffix}" @property def parameters(self) -> List[FormalParameterView]: """ The owning member's formal parameters, none for a command """ - return [] if isinstance(self.owner, CommandView) else self.owner.parameters + return [] if self.is_command_hook else self.owner.parameters class EventView: @@ -494,18 +510,18 @@ def overflow_hooks(self) -> List[OverflowHookView]: behavior, in that order """ hooks = [ - OverflowHookView(port, "overflowHook") + OverflowHookView(port) for port in self.input_ports if isinstance(port.port.kind, fpp.GeneralKind.AsyncInput) and port.port.kind.queue_full == fpp.ast.QueueFull.Hook ] hooks += [ - OverflowHookView(port, "overflowHook") + OverflowHookView(port) for port in self.internal_ports if port.port.queue_full == fpp.ast.QueueFull.Hook ] hooks += [ - OverflowHookView(command, "cmdOverflowHook") + OverflowHookView(command) for command in self.commands if isinstance(command.command.kind, fpp.NonParamKind.Async) and command.command.kind.queue_full == fpp.ast.QueueFull.Hook @@ -598,10 +614,10 @@ def commands(self) -> List[CommandView]: @property def handlers(self) -> List[HandlerView]: - """ Every component member F Prime declares a handler for, in generation order + """ Every handler F Prime declares, members first and overflow hooks last, in generation order Every family gets the same forwarding C++ override and a stub in the Python template (a command's - stub also sends its response), so the generators walk this one list rather than each family in turn. + stub, and its overflow hook's, also sends a response), so the generators walk this one list rather than each family in turn. """ handlers: List[HandlerView] = list(self.input_ports) handlers += self.internal_ports From ce9acd84a77cae3aba124169d6f8f8b58805ad54 Mon Sep 17 00:00:00 2001 From: lestarch-autobot Date: Thu, 29 Oct 2026 17:33:40 +0000 Subject: [PATCH 6/8] Extend overflow hook tests Cover the GIL taken by each hook, the init check, an async command without hook, and avoid the 'msg' name that shadows a local in F Prime's internal port invoke. Co-authored-by: M Starch --- tests/fixtures/Hooks.fpp | 5 +++-- tests/test_generator.py | 34 ++++++++++++++++++++++++++-------- 2 files changed, 29 insertions(+), 10 deletions(-) diff --git a/tests/fixtures/Hooks.fpp b/tests/fixtures/Hooks.fpp index 125f32d..3012683 100644 --- a/tests/fixtures/Hooks.fpp +++ b/tests/fixtures/Hooks.fpp @@ -1,15 +1,16 @@ module Fixture { @ Port with a string argument - port Text(msg: string size 16) + port Text(note: string size 16) @ A component asking for an overflow hook on every member that can have one @ fprime-python queued component PyHooks { async input port textIn: Text hook async input port dropIn: Text drop - internal port poke(msg: string size 5) hook + internal port poke(note: string size 5) hook async command DO(n: U32) hook + async command PLAIN command recv port cmdIn command reg port cmdRegOut command resp port cmdResponseOut diff --git a/tests/test_generator.py b/tests/test_generator.py index df3e0f1..98e42f5 100644 --- a/tests/test_generator.py +++ b/tests/test_generator.py @@ -130,24 +130,42 @@ def hooks(tmp_path: Path) -> Dict[str, str]: def test_overflow_hooks_are_forwarded_into_python(hooks: Dict[str, str]) -> None: header = hooks["PyHooks.hpp"] assert "void textIn_overflowHook(" in header - assert "void poke_overflowHook(const Fw::InternalInterfaceString& msg) override;" in header + assert "void poke_overflowHook(const Fw::InternalInterfaceString& note) override;" in header assert "void DO_cmdOverflowHook(" in header source = hooks["PyHooks.cpp"] - assert 'm_self.attr("textIn_overflowHook")(portNum, msg);' in source - assert 'm_self.attr("poke_overflowHook")(msg);' in source - # A command's hook gets the opcode and sequence number, not the command's arguments - assert 'm_self.attr("DO_cmdOverflowHook")(opCode, cmdSeq);' in source + calls = [ + 'm_self.attr("textIn_overflowHook")(portNum, note);', + 'm_self.attr("poke_overflowHook")(note);', + # A command's hook gets the opcode and sequence number, not the command's arguments + 'm_self.attr("DO_cmdOverflowHook")(opCode, cmdSeq);', + ] + for call in calls: + # The hook runs on the caller's thread, so it must take the GIL itself before calling Python + body = source[source.rindex("PyHooks ::", 0, source.index(call)):source.index(call)] + assert "pybind11::gil_scoped_acquire acquired{};" in body + + +def test_missing_overflow_hooks_are_reported_at_init(hooks: Dict[str, str]) -> None: + source = hooks["PyHooks.cpp"] + init = source[source.index('.attr("_init_ac")(this);'):source.index("PyHooksComponentBase::init(")] + for hook in ("textIn_overflowHook", "poke_overflowHook", "DO_cmdOverflowHook"): + assert f'if (!pybind11::hasattr(this->m_self, "{hook}")) {{' in init + assert f'throw pybind11::attribute_error("PyHooks does not implement overflow hook {hook}");' in init def test_overflow_hooks_get_python_stubs(hooks: Dict[str, str]) -> None: template = hooks["PyHooks.template.py"] - assert "def textIn_overflowHook(self, portNum, msg):" in template - assert "def poke_overflowHook(self, msg):" in template - hook = template[template.index("def DO_cmdOverflowHook(self, opCode, cmdSeq):"):] + assert "def textIn_overflowHook(self, portNum, note):" in template + assert "def poke_overflowHook(self, note):" in template + start = template.index("def DO_cmdOverflowHook(self, opCode, cmdSeq):") + end = template.find("\n def ", start) + hook = template[start:end if end != -1 else len(template)] assert "self.cmdResponse_out(opCode, cmdSeq, fprime_py.Fw.CmdResponse(fprime_py.Fw.CmdResponse.T.BUSY))" in hook def test_members_without_hook_get_no_overflow_hook(files: Dict[str, str], hooks: Dict[str, str]) -> None: assert "dropIn_overflowHook" not in hooks["PyHooks.cpp"] + assert "PLAIN_cmdOverflowHook" not in hooks["PyHooks.cpp"] + assert "PLAIN_cmdOverflowHook" not in hooks["PyHooks.template.py"] assert "overflowHook" not in files["PyQueued.cpp"] assert "overflowHook" not in files["PyQueued.template.py"] From 6040e63e6f68517e3ad493c2092a21139a9a3e3d Mon Sep 17 00:00:00 2001 From: lestarch-autobot Date: Thu, 29 Oct 2026 17:33:40 +0000 Subject: [PATCH 7/8] Document overflow hook threading, ownership and startup check Co-authored-by: M Starch --- README.md | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 965e35d..07b6363 100644 --- a/README.md +++ b/README.md @@ -91,18 +91,31 @@ message, and the message is dropped once the hook returns. | Member | Hook | Arguments | | --- | --- | --- | | `async input port textIn: Text hook` | `textIn_overflowHook` | `portNum` and the port's arguments | -| `internal port poke(msg: string size 5) hook` | `poke_overflowHook` | the internal port's arguments | +| `internal port poke(note: string size 5) hook` | `poke_overflowHook` | the internal port's arguments | | `async command DO(n: U32) hook` | `DO_cmdOverflowHook` | `opCode` and `cmdSeq` only | -The hook runs on the thread of the caller that found the queue full, not on the component's thread, so keep it short -and do not block in it. F Prime does not answer a command it dropped: the template's command hook sends `BUSY` so the -ground is not left waiting. +Every hook must be implemented: a component whose Python class lacks one raises `AttributeError` from its generated +`init`, at startup rather than at the first overflow. The template has a stub for each. + +- **Thread.** The hook runs on the thread of the caller that found the queue full, not on the component's thread. + Before any Python runs, that thread must take the GIL, which the component's own busy thread often holds, so even an + empty hook can delay the caller. Keep the hook short, never block in it, size the queue so that overflow stays the + exception, and prefer `drop` where the caller's timing matters. +- **Concurrency.** The hook can run while one of the component's handlers is part-way through on the component's + thread, so it must not rely on state that a handler updates in several steps. +- **Ownership.** F Prime does not count or report a hooked drop. The hook owns the dropped message: if a port carries + an `Fw::Buffer`, the hook must return or deallocate it. +- **Commands.** F Prime does not answer a command it dropped, so a command hook must send a response. The template's + stub sends `BUSY`: ```python def DO_cmdOverflowHook(self, opCode, cmdSeq): self.cmdResponse_out(opCode, cmdSeq, fprime_py.Fw.CmdResponse(fprime_py.Fw.CmdResponse.T.BUSY)) ``` +An exception raised in a hook is not caught by the generated C++, as for any handler (see [Strings](#strings)), but +here it reaches the caller's thread. + ## Setting a Deployment Up With Python Bindings The deployment topology that instantiates Python components must also carry the `@ fprime-python` annotation. Only an From 811ea7f1de1de0b8d792d9dfb6f08a07c74ef1e0 Mon Sep 17 00:00:00 2001 From: lestarch-autobot Date: Thu, 29 Oct 2026 17:42:05 +0000 Subject: [PATCH 8/8] Check overflow hooks on the class and release the Python object before throwing Looking the hook up on the class keeps the base class's fall-through to the C++ object from satisfying it; releasing m_self first keeps it from outliving the interpreter when teardown is skipped. Pin the generated hook comments in the tests and rewrap long docstrings. Co-authored-by: M Starch --- src/fprime_python/autocode/component_generator.py | 7 +++++-- src/fprime_python/autocode/view.py | 11 ++++++----- tests/test_generator.py | 8 ++++++-- 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/src/fprime_python/autocode/component_generator.py b/src/fprime_python/autocode/component_generator.py index b567903..ee63270 100644 --- a/src/fprime_python/autocode/component_generator.py +++ b/src/fprime_python/autocode/component_generator.py @@ -353,9 +353,12 @@ def _write_init(self, body: Body) -> None: 'Call auto-coded initialization function storing the C++ "this" object for Python' ) body.line(f'this->{SELF_MEMBER}.attr("_init_ac")(this);') - # A hook runs only when the queue fills, so a missing one is reported at startup, not first overflow + # A hook runs only when the queue fills, so a missing one is reported at startup, not first overflow. + # It is looked up on the class, which the base class's fall-through to the C++ object cannot satisfy, + # and the Python object is released first so it cannot outlive the interpreter if teardown is skipped for hook in self.component.overflow_hooks: - with body.if_(f'!pybind11::hasattr(this->{SELF_MEMBER}, "{hook.handler_name}")'): + with body.if_(f'!pybind11::hasattr(pybind11::type::of(this->{SELF_MEMBER}), "{hook.handler_name}")'): + body.line(f"this->{SELF_MEMBER} = pybind11::none();") body.line( f'throw pybind11::attribute_error("{self.name} does not implement overflow hook' f' {hook.handler_name}");' diff --git a/src/fprime_python/autocode/view.py b/src/fprime_python/autocode/view.py index 1b89b55..2733e13 100644 --- a/src/fprime_python/autocode/view.py +++ b/src/fprime_python/autocode/view.py @@ -113,9 +113,9 @@ class HandlerView(ABC): A handler's signature is a fixed prefix -- the port number for a general port, the opcode and the command sequence number for a command, nothing for an internal port -- followed by its formal parameters: - the member's own, or none for a command's overflow hook. The generated C++ override, the call it forwards into Python, and the stub in the - Python template all have to agree on that signature, so it is spelled once here and each generator - reads it back rather than repeating the prefix. + the member's own, or none for a command's overflow hook. The generated C++ override, the call it forwards + into Python, and the stub in the Python template all have to agree on that signature, so it is spelled once + here and each generator reads it back rather than repeating the prefix. """ #: The handler's fixed leading parameters, as (C++ type, name, doc comment) triples @@ -338,7 +338,7 @@ def is_command_hook(self) -> bool: @property def description(self) -> str: - """ The hook as F Prime's own generated comment names it """ + """ What the hook is, as it reads in a generated comment """ return f"Overflow hook for {self.owner.HANDLER_KIND} {self.name}" @property @@ -617,7 +617,8 @@ def handlers(self) -> List[HandlerView]: """ Every handler F Prime declares, members first and overflow hooks last, in generation order Every family gets the same forwarding C++ override and a stub in the Python template (a command's - stub, and its overflow hook's, also sends a response), so the generators walk this one list rather than each family in turn. + stub, and its overflow hook's, also sends a response), so the generators walk this one list rather than + each family in turn. """ handlers: List[HandlerView] = list(self.input_ports) handlers += self.internal_ports diff --git a/tests/test_generator.py b/tests/test_generator.py index 98e42f5..67cae6f 100644 --- a/tests/test_generator.py +++ b/tests/test_generator.py @@ -129,6 +129,7 @@ def hooks(tmp_path: Path) -> Dict[str, str]: def test_overflow_hooks_are_forwarded_into_python(hooks: Dict[str, str]) -> None: header = hooks["PyHooks.hpp"] + assert "//! Overflow hook for command DO" in header assert "void textIn_overflowHook(" in header assert "void poke_overflowHook(const Fw::InternalInterfaceString& note) override;" in header assert "void DO_cmdOverflowHook(" in header @@ -149,14 +150,17 @@ def test_missing_overflow_hooks_are_reported_at_init(hooks: Dict[str, str]) -> N source = hooks["PyHooks.cpp"] init = source[source.index('.attr("_init_ac")(this);'):source.index("PyHooksComponentBase::init(")] for hook in ("textIn_overflowHook", "poke_overflowHook", "DO_cmdOverflowHook"): - assert f'if (!pybind11::hasattr(this->m_self, "{hook}")) {{' in init - assert f'throw pybind11::attribute_error("PyHooks does not implement overflow hook {hook}");' in init + check = init[init.index(f'if (!pybind11::hasattr(pybind11::type::of(this->m_self), "{hook}")) {{'):] + check = check[:check.index("}")] + assert "this->m_self = pybind11::none();" in check + assert f'throw pybind11::attribute_error("PyHooks does not implement overflow hook {hook}");' in check def test_overflow_hooks_get_python_stubs(hooks: Dict[str, str]) -> None: template = hooks["PyHooks.template.py"] assert "def textIn_overflowHook(self, portNum, note):" in template assert "def poke_overflowHook(self, note):" in template + assert '""" Handle the DO command overflow hook """' in template start = template.index("def DO_cmdOverflowHook(self, opCode, cmdSeq):") end = template.find("\n def ", start) hook = template[start:end if end != -1 else len(template)]