diff --git a/README.md b/README.md index cc77282..07b6363 100644 --- a/README.md +++ b/README.md @@ -69,19 +69,53 @@ Once finished, the python bindings will be autocoded and included in the next bu as shown below). This will also produce a `.template.py` file in the component folder as a basic template for implementing components in python. -Every port, command, event, telemetry channel, parameter and internal port of the component is bound. Four -component features cannot be bound yet, because F Prime declares a handler for each that has no Python equivalent: +Every port, command, event, telemetry channel, parameter and internal port of the component is bound, as is the +overflow hook of each member with `hook` queue-full behavior (see below). Three component features cannot be bound +yet, because F Prime declares a handler for each that has no Python equivalent: | Feature | Why | What happens | | --- | --- | --- | | serial port | its handler takes a serialization buffer | rejected with an error naming the port | | state machine instance | its actions and guards take a state machine id and a signal | rejected with an error naming the component | | data product container | its handler takes a `Fw::DpContainer` | rejected with an error naming the component | -| `hook` queue-full behavior | its overflow hook takes the message that could not be queued | rejected with an error naming the component | In each case the autocoder refuses rather than generating a class that would leave the handler unimplemented and fail to link. Drop the `@ fprime-python` annotation to implement such a component in C++ instead. +### Queue-Full Overflow Hooks + +An async input port, internal port or async command declared with `hook` queue-full behavior gets an overflow hook +in Python next to its handler. When the component's queue is full, F Prime calls the hook instead of queuing the +message, and the message is dropped once the hook returns. + +| Member | Hook | Arguments | +| --- | --- | --- | +| `async input port textIn: Text hook` | `textIn_overflowHook` | `portNum` and the port's arguments | +| `internal port poke(note: string size 5) hook` | `poke_overflowHook` | the internal port's arguments | +| `async command DO(n: U32) hook` | `DO_cmdOverflowHook` | `opCode` and `cmdSeq` only | + +Every hook must be implemented: a component whose Python class lacks one raises `AttributeError` from its generated +`init`, at startup rather than at the first overflow. The template has a stub for each. + +- **Thread.** The hook runs on the thread of the caller that found the queue full, not on the component's thread. + Before any Python runs, that thread must take the GIL, which the component's own busy thread often holds, so even an + empty hook can delay the caller. Keep the hook short, never block in it, size the queue so that overflow stays the + exception, and prefer `drop` where the caller's timing matters. +- **Concurrency.** The hook can run while one of the component's handlers is part-way through on the component's + thread, so it must not rely on state that a handler updates in several steps. +- **Ownership.** F Prime does not count or report a hooked drop. The hook owns the dropped message: if a port carries + an `Fw::Buffer`, the hook must return or deallocate it. +- **Commands.** F Prime does not answer a command it dropped, so a command hook must send a response. The template's + stub sends `BUSY`: + +```python +def DO_cmdOverflowHook(self, opCode, cmdSeq): + self.cmdResponse_out(opCode, cmdSeq, fprime_py.Fw.CmdResponse(fprime_py.Fw.CmdResponse.T.BUSY)) +``` + +An exception raised in a hook is not caught by the generated C++, as for any handler (see [Strings](#strings)), but +here it reaches the caller's thread. + ## Setting a Deployment Up With Python Bindings The deployment topology that instantiates Python components must also carry the `@ fprime-python` annotation. Only an diff --git a/src/fprime_python/autocode/component_generator.py b/src/fprime_python/autocode/component_generator.py index 0661e1b..ee63270 100644 --- a/src/fprime_python/autocode/component_generator.py +++ b/src/fprime_python/autocode/component_generator.py @@ -18,7 +18,14 @@ from .binding_generator import BindingGenerator, expression_chain, standard_def from .constants import SELF_MEMBER, SUPPORT_HEADER, TOOL_NAME from .include import IncludeManager -from .view import CommandView, ComponentView, FormalParameterView, HandlerView, ParameterView +from .view import ( + CommandView, + ComponentView, + FormalParameterView, + HandlerView, + OverflowHookView, + ParameterView, +) #: Telemetry write bindings keep the default timestamp argument F Prime declares, so that Python callers @@ -277,11 +284,11 @@ class {name}({name}Base): pass ''' -#: Stub for one command handler in the Python implementation template +#: Stub for one command handler or command overflow hook in the Python implementation template PYTHON_COMMAND_HANDLER_TEMPLATE = ''' def {handler}(self{args}): - """ Handle the {name} command """ - # TODO: Implement command handler - self.cmdResponse_out({prefix}, fprime_py.Fw.CmdResponse(fprime_py.Fw.CmdResponse.T.OK)) + """ Handle the {name} {kind} """ + # TODO: Implement {kind} handler + self.cmdResponse_out({prefix}, fprime_py.Fw.CmdResponse(fprime_py.Fw.CmdResponse.T.{response})) ''' @@ -346,6 +353,16 @@ def _write_init(self, body: Body) -> None: 'Call auto-coded initialization function storing the C++ "this" object for Python' ) body.line(f'this->{SELF_MEMBER}.attr("_init_ac")(this);') + # A hook runs only when the queue fills, so a missing one is reported at startup, not first overflow. + # It is looked up on the class, which the base class's fall-through to the C++ object cannot satisfy, + # and the Python object is released first so it cannot outlive the interpreter if teardown is skipped + for hook in self.component.overflow_hooks: + with body.if_(f'!pybind11::hasattr(pybind11::type::of(this->{SELF_MEMBER}), "{hook.handler_name}")'): + body.line(f"this->{SELF_MEMBER} = pybind11::none();") + body.line( + f'throw pybind11::attribute_error("{self.name} does not implement overflow hook' + f' {hook.handler_name}");' + ) body.comment("Continue the standard initialization of F Prime") arguments = ", ".join(name for _, name, _ in self._init_params()) body.line(f"{self.component.base_class}::init({arguments});") @@ -436,7 +453,7 @@ def _write_handlers(self, cls: ClassBuilder) -> None: ret=handler.handler_return_type, params=handler.handler_parameters, override=True, - comment=f"Handler for {handler.HANDLER_KIND} {handler.name}", + comment=handler.description, ) self._write_forwarding_call(override.body, handler) @@ -556,14 +573,17 @@ def python_implementation(self) -> str: for handler in self.component.handlers: args = "".join(f", {name}" for name in handler.handler_arguments) # A command handler is not left to fall off the end: the ground waits for a response, so the - # stub sends one - if isinstance(handler, CommandView): + # stub sends one. A command's overflow hook answers too, since F Prime dropped the command unanswered + overflowed = isinstance(handler, OverflowHookView) and handler.is_command_hook + if isinstance(handler, CommandView) or overflowed: stubs.append( PYTHON_COMMAND_HANDLER_TEMPLATE.format( handler=handler.handler_name, name=handler.name, + kind=handler.HANDLER_KIND, args=args, prefix=", ".join(name for _, name, _ in handler.HANDLER_PREFIX), + response="BUSY" if overflowed else "OK", ) ) continue diff --git a/src/fprime_python/autocode/view.py b/src/fprime_python/autocode/view.py index a7ec6ce..2733e13 100644 --- a/src/fprime_python/autocode/view.py +++ b/src/fprime_python/autocode/view.py @@ -109,13 +109,13 @@ class UnsupportedModelError(Exception): class HandlerView(ABC): - """ One component member that F Prime declares a pure virtual handler for + """ One pure virtual handler F Prime declares for a component member A handler's signature is a fixed prefix -- the port number for a general port, the opcode and the - command sequence number for a command, nothing for an internal port -- followed by the member's own - formal parameters. The generated C++ override, the call it forwards into Python, and the stub in the - Python template all have to agree on that signature, so it is spelled once here and each generator - reads it back rather than repeating the prefix. + command sequence number for a command, nothing for an internal port -- followed by its formal parameters: + the member's own, or none for a command's overflow hook. The generated C++ override, the call it forwards + into Python, and the stub in the Python template all have to agree on that signature, so it is spelled once + here and each generator reads it back rather than repeating the prefix. """ #: The handler's fixed leading parameters, as (C++ type, name, doc comment) triples @@ -134,6 +134,11 @@ def name(self) -> str: def handler_name(self) -> str: """ The name of the handler F Prime declares for this member """ + @property + def description(self) -> str: + """ What the handler is, as it reads in a generated comment """ + return f"Handler for {self.HANDLER_KIND} {self.name}" + @property @abstractmethod def parameters(self) -> List[FormalParameterView]: @@ -312,6 +317,47 @@ def parameters(self) -> List[FormalParameterView]: ] +class OverflowHookView(HandlerView): + """ The overflow hook F Prime declares for a member with `hook` queue-full behavior + + F Prime calls it instead of the handler when the message cannot be queued. It takes the handler's arguments, + except that a command's hook gets only the opcode and the command sequence number. The handler prefix is the + owner's, so it is copied onto the instance rather than declared by the class. + """ + + def __init__(self, owner: HandlerView) -> None: + """ Wrap the hook of a port, internal port or command """ + self.owner = owner + self.HANDLER_PREFIX = owner.HANDLER_PREFIX + self.HANDLER_KIND = f"{owner.HANDLER_KIND} overflow hook" + + @property + def is_command_hook(self) -> bool: + """ Whether the hook is a command's, which takes no command arguments and must answer the command """ + return isinstance(self.owner, CommandView) + + @property + def description(self) -> str: + """ What the hook is, as it reads in a generated comment """ + return f"Overflow hook for {self.owner.HANDLER_KIND} {self.name}" + + @property + def name(self) -> str: + """ The owning member's name as written in the model """ + return self.owner.name + + @property + def handler_name(self) -> str: + """ The name of the overflow hook F Prime declares for the owning member """ + suffix = "cmdOverflowHook" if self.is_command_hook else "overflowHook" + return f"{self.owner.name}_{suffix}" + + @property + def parameters(self) -> List[FormalParameterView]: + """ The owning member's formal parameters, none for a command """ + return [] if self.is_command_hook else self.owner.parameters + + class EventView: """ One event of a component """ @@ -459,49 +505,43 @@ def has_commands(self) -> bool: return bool(self.commands) @property - def has_queue_full_hooks(self) -> bool: - """ Whether anything on the component declares `hook` queue-full behavior - - `hook` makes F Prime declare a pure virtual overflow hook -- `_overflowHook`, - `_overflowHook` or `_cmdOverflowHook` -- alongside the handler. Async input ports, - internal ports and async commands can each ask for one. + def overflow_hooks(self) -> List[OverflowHookView]: + """ The overflow hooks of the async input ports, internal ports and async commands with `hook` queue-full + behavior, in that order """ - for port in self.component.port_map.values(): - if isinstance(port, fpp.PortInstance.General): - kind = port.kind - if ( - isinstance(kind, fpp.GeneralKind.AsyncInput) - and kind.queue_full == fpp.ast.QueueFull.Hook - ): - return True - elif ( - isinstance(port, fpp.PortInstance.Internal) - and port.queue_full == fpp.ast.QueueFull.Hook - ): - return True - return any( - isinstance(command.command.kind, fpp.NonParamKind.Async) - and command.command.kind.queue_full == fpp.ast.QueueFull.Hook + hooks = [ + OverflowHookView(port) + for port in self.input_ports + if isinstance(port.port.kind, fpp.GeneralKind.AsyncInput) + and port.port.kind.queue_full == fpp.ast.QueueFull.Hook + ] + hooks += [ + OverflowHookView(port) + for port in self.internal_ports + if port.port.queue_full == fpp.ast.QueueFull.Hook + ] + hooks += [ + OverflowHookView(command) for command in self.commands - ) + if isinstance(command.command.kind, fpp.NonParamKind.Async) + and command.command.kind.queue_full == fpp.ast.QueueFull.Hook + ] + return hooks def check_supported(self) -> None: """ Reject a component whose features this autocoder cannot bind - F Prime declares a pure virtual handler for every state machine action and guard, for every data - product container, and for every `hook` queue-full overflow. None has a Python counterpart yet -- - an action takes a state machine id and a signal, a container handler takes a `Fw::DpContainer`, and - an overflow hook takes the message that could not be queued -- so generating the component anyway - would leave an abstract class and fail at link time, a long way from the cause. + F Prime declares a pure virtual handler for every state machine action and guard, and for every data + product container. Neither has a Python counterpart yet -- an action takes a state machine id and a + signal, and a container handler takes a `Fw::DpContainer` -- so generating the component anyway would + leave an abstract class and fail at link time, a long way from the cause. Raises: - UnsupportedModelError: The component has state machine instances, data products or a `hook` - queue-full declaration + UnsupportedModelError: The component has state machine instances or data products """ unsupported = [ ("state machine instances", self.component.has_state_machine_instances), ("data products", self.component.has_data_products), - ("`hook` queue-full behavior", self.has_queue_full_hooks), ] for feature, present in unsupported: if present: @@ -574,14 +614,16 @@ def commands(self) -> List[CommandView]: @property def handlers(self) -> List[HandlerView]: - """ Every component member F Prime declares a handler for, in generation order + """ Every handler F Prime declares, members first and overflow hooks last, in generation order - The three families get the same forwarding C++ override and a stub in the Python template (a command's - stub also sends its response), so the generators walk this one list rather than each family in turn. + Every family gets the same forwarding C++ override and a stub in the Python template (a command's + stub, and its overflow hook's, also sends a response), so the generators walk this one list rather than + each family in turn. """ handlers: List[HandlerView] = list(self.input_ports) handlers += self.internal_ports handlers += self.commands + handlers += self.overflow_hooks return handlers @property diff --git a/tests/fixtures/Config.fpp b/tests/fixtures/Config.fpp index f9b5e25..0828040 100644 --- a/tests/fixtures/Config.fpp +++ b/tests/fixtures/Config.fpp @@ -6,15 +6,21 @@ type FwSizeType = U64 type FwEventIdType = U32 type FwChanIdType = U32 +type FwOpcodeType = U32 -@ Stand-ins for the framework ports that events and telemetry need +@ Stand-ins for the framework ports that commands, events and telemetry need module Fw { type Time type LogBuffer type TlmBuffer + type CmdArgBuffer + enum CmdResponse { OK } enum LogSeverity { ACTIVITY_LO } port Log($id: FwEventIdType, ref timeTag: Fw.Time, $severity: LogSeverity, ref args: LogBuffer) port LogText($id: FwEventIdType, ref timeTag: Fw.Time, $severity: LogSeverity, ref $text: string size 256) port Time(ref $time: Fw.Time) + port Cmd(opCode: FwOpcodeType, cmdSeq: U32, ref args: CmdArgBuffer) + port CmdReg(opCode: FwOpcodeType) + port CmdResponse(opCode: FwOpcodeType, cmdSeq: U32, response: CmdResponse) port Tlm($id: FwChanIdType, ref timeTag: Fw.Time, ref val: TlmBuffer) } diff --git a/tests/fixtures/Hooks.fpp b/tests/fixtures/Hooks.fpp new file mode 100644 index 0000000..3012683 --- /dev/null +++ b/tests/fixtures/Hooks.fpp @@ -0,0 +1,19 @@ +module Fixture { + + @ Port with a string argument + port Text(note: string size 16) + + @ A component asking for an overflow hook on every member that can have one + @ fprime-python + queued component PyHooks { + async input port textIn: Text hook + async input port dropIn: Text drop + internal port poke(note: string size 5) hook + async command DO(n: U32) hook + async command PLAIN + command recv port cmdIn + command reg port cmdRegOut + command resp port cmdResponseOut + } + +} diff --git a/tests/test_generator.py b/tests/test_generator.py index b0e961a..67cae6f 100644 --- a/tests/test_generator.py +++ b/tests/test_generator.py @@ -120,3 +120,56 @@ def test_sized_string_return_is_checked_against_its_declared_size(files: Dict[st assert check in source assert source.index("Fw::String result = return_value.cast();") < source.index(check) assert source.index(check) < source.index("return result;") + + +@pytest.fixture +def hooks(tmp_path: Path) -> Dict[str, str]: + return generate("Hooks.fpp", tmp_path) + + +def test_overflow_hooks_are_forwarded_into_python(hooks: Dict[str, str]) -> None: + header = hooks["PyHooks.hpp"] + assert "//! Overflow hook for command DO" in header + assert "void textIn_overflowHook(" in header + assert "void poke_overflowHook(const Fw::InternalInterfaceString& note) override;" in header + assert "void DO_cmdOverflowHook(" in header + source = hooks["PyHooks.cpp"] + calls = [ + 'm_self.attr("textIn_overflowHook")(portNum, note);', + 'm_self.attr("poke_overflowHook")(note);', + # A command's hook gets the opcode and sequence number, not the command's arguments + 'm_self.attr("DO_cmdOverflowHook")(opCode, cmdSeq);', + ] + for call in calls: + # The hook runs on the caller's thread, so it must take the GIL itself before calling Python + body = source[source.rindex("PyHooks ::", 0, source.index(call)):source.index(call)] + assert "pybind11::gil_scoped_acquire acquired{};" in body + + +def test_missing_overflow_hooks_are_reported_at_init(hooks: Dict[str, str]) -> None: + source = hooks["PyHooks.cpp"] + init = source[source.index('.attr("_init_ac")(this);'):source.index("PyHooksComponentBase::init(")] + for hook in ("textIn_overflowHook", "poke_overflowHook", "DO_cmdOverflowHook"): + check = init[init.index(f'if (!pybind11::hasattr(pybind11::type::of(this->m_self), "{hook}")) {{'):] + check = check[:check.index("}")] + assert "this->m_self = pybind11::none();" in check + assert f'throw pybind11::attribute_error("PyHooks does not implement overflow hook {hook}");' in check + + +def test_overflow_hooks_get_python_stubs(hooks: Dict[str, str]) -> None: + template = hooks["PyHooks.template.py"] + assert "def textIn_overflowHook(self, portNum, note):" in template + assert "def poke_overflowHook(self, note):" in template + assert '""" Handle the DO command overflow hook """' in template + start = template.index("def DO_cmdOverflowHook(self, opCode, cmdSeq):") + end = template.find("\n def ", start) + hook = template[start:end if end != -1 else len(template)] + assert "self.cmdResponse_out(opCode, cmdSeq, fprime_py.Fw.CmdResponse(fprime_py.Fw.CmdResponse.T.BUSY))" in hook + + +def test_members_without_hook_get_no_overflow_hook(files: Dict[str, str], hooks: Dict[str, str]) -> None: + assert "dropIn_overflowHook" not in hooks["PyHooks.cpp"] + assert "PLAIN_cmdOverflowHook" not in hooks["PyHooks.cpp"] + assert "PLAIN_cmdOverflowHook" not in hooks["PyHooks.template.py"] + assert "overflowHook" not in files["PyQueued.cpp"] + assert "overflowHook" not in files["PyQueued.template.py"]