diff --git a/.github/ISSUE_TEMPLATE/device-test-report.md b/.github/ISSUE_TEMPLATE/device-test-report.md new file mode 100644 index 00000000000..438a2f8babb --- /dev/null +++ b/.github/ISSUE_TEMPLATE/device-test-report.md @@ -0,0 +1,46 @@ +--- +name: Device hardware test report +about: Tell us what worked or failed when using PyLabRobot on a real device. +title: "[Hardware test] Vendor model — Works / Partly works / Does not work" +labels: '' +assignees: '' +--- + + + +### Device and environment + +- Manufacturer and exact model: +- Firmware version (or unknown): +- PyLabRobot version or Git commit (`git rev-parse HEAD`): +- Operating system and Python version: +- Connection type and settings (USB, serial, TCP, adapters, etc.): +- Relevant hardware configuration, accessories, and labware: +- Date tested: +- Guide followed and related issue / pull request (if any): + +### Result on real hardware + +Overall outcome: Works / Partly works / Does not work + + + +### Reproduction code + + + +```python + +``` + +### Evidence + + + +### Limitations and follow-up + + diff --git a/docs/_exts/plr_devices/data.py b/docs/_exts/plr_devices/data.py index 5a2113d56eb..f0c5cd4823a 100644 --- a/docs/_exts/plr_devices/data.py +++ b/docs/_exts/plr_devices/data.py @@ -85,7 +85,7 @@ class DeviceRegistryError(ValueError): } STATUS_DESCRIPTIONS = { - "wip": "Work in progress.", + "wip": "Work in progress, including models awaiting initial hardware verification.", "basic": "Core functionality is available.", "mostly": "Most capabilities are available, but some known commands are still missing.", "full": "Comprehensive support (at least 90% of capabilities), with documentation.", @@ -110,6 +110,7 @@ class DeviceRegistryError(ValueError): "manager", "oem", "notes", + "needs_hardware_testing", ) _ALL_FIELDS = set(REQUIRED_FIELDS) | set(OPTIONAL_FIELDS) @@ -176,6 +177,9 @@ def _validate(device: Any, index: int, seen_ids: Dict[str, int], path: Path) -> f"{where} ({device_id}): api_version {api_version!r} is not one of {', '.join(API_VERSIONS)}" ) + if "needs_hardware_testing" in device and not isinstance(device["needs_hardware_testing"], bool): + raise DeviceRegistryError(f"{where} ({device_id}): needs_hardware_testing must be a boolean") + for field in ("manager", "oem"): value = device.get(field) if value is not None and not str(value).startswith(("http://", "https://")): @@ -199,13 +203,15 @@ def _validate(device: Any, index: int, seen_ids: Dict[str, int], path: Path) -> model_where = f"{where} ({device_id}) models[{model_index}]" if not isinstance(model, dict): raise DeviceRegistryError(f"{model_where}: model must be an object") - unknown_model_fields = sorted(set(model) - {"name", "status"}) + unknown_model_fields = sorted(set(model) - {"name", "status", "needs_hardware_testing"}) if unknown_model_fields: raise DeviceRegistryError( f"{model_where}: unknown field(s): {', '.join(unknown_model_fields)}" ) if not isinstance(model.get("name"), str) or not model["name"]: raise DeviceRegistryError(f"{model_where}: name must be a non-empty string") + if "needs_hardware_testing" in model and not isinstance(model["needs_hardware_testing"], bool): + raise DeviceRegistryError(f"{model_where}: needs_hardware_testing must be a boolean") model_status = model.get("status") if model_status is not None and model_status not in STATUSES: raise DeviceRegistryError( @@ -259,12 +265,31 @@ def get_device(app, device_id: str) -> Optional[Device]: return None +def needs_hardware_testing(device: Device, model: Optional[Dict[str, Any]] = None) -> bool: + """Whether a device or model needs testing; models inherit the device flag by default.""" + if model is not None: + return bool(model.get("needs_hardware_testing", device.get("needs_hardware_testing", False))) + if device.get("models"): + return any(needs_hardware_testing(device, entry) for entry in device["models"]) + return bool(device.get("needs_hardware_testing", False)) + + def filter_devices(devices: Sequence[Device], filters: Dict[str, str]) -> List[Device]: """Keep devices matching every filter.""" def matches(device: Device, field: str, wanted: str) -> bool: + if field == "needs_hardware_testing": + return needs_hardware_testing(device) == (wanted.lower() == "true") if field == "capabilities": return wanted.lower() in {c.lower() for c in device.get("capabilities", [])} return str(device.get(field, "")).lower() == wanted.lower() - return [d for d in devices if all(matches(d, f, w) for f, w in filters.items() if w)] + selected = [d for d in devices if all(matches(d, f, w) for f, w in filters.items() if w)] + if filters.get("needs_hardware_testing", "").lower() == "true": + # Keep the full registry intact for other tables and cards in the same build. + return [ + Device({**d, "models": [m for m in d["models"] if needs_hardware_testing(d, m)]}) + if d.get("models") else d + for d in selected + ] + return selected diff --git a/docs/_exts/plr_devices/directive.py b/docs/_exts/plr_devices/directive.py index a948f7102e1..a73b0b90929 100644 --- a/docs/_exts/plr_devices/directive.py +++ b/docs/_exts/plr_devices/directive.py @@ -68,6 +68,7 @@ class DeviceTable(Directive): "status": directives.unchanged, "search": directives.unchanged, "filters": directives.unchanged, + "needs-hardware-testing": directives.flag, } def run(self): @@ -80,6 +81,8 @@ def run(self): } node["search"] = _flag(self.options.get("search")) node["filters_ui"] = _flag(self.options.get("filters")) + if "needs-hardware-testing" in self.options: + node["filters"]["needs_hardware_testing"] = "true" return [node] diff --git a/docs/_static/devices.json b/docs/_static/devices.json index 61fe3960e64..21fb4276af3 100644 --- a/docs/_static/devices.json +++ b/docs/_static/devices.json @@ -231,13 +231,14 @@ }, { "id": "big-bear-orbital-shaker", + "needs_hardware_testing": true, "vendor": "BigBear", "name": "Orbital Shaker", "kind": "shaker", "capabilities": [ "shaking" ], - "status": "mostly", + "status": "wip", "api": "pylabrobot.big_bear.BigBearOrbitalShaker", "api_version": "v1", "code_slug": "big_bear", @@ -344,6 +345,7 @@ }, { "id": "cole-parmer-genogrinder", + "needs_hardware_testing": true, "vendor": "Cole Parmer", "name": "GenoGrinder", "kind": "shaker", @@ -351,7 +353,7 @@ "shaking", "grinding" ], - "status": "mostly", + "status": "wip", "api": "pylabrobot.cole_parmer.GenoGrinder", "api_version": "v1", "code_slug": "cole_parmer", @@ -384,13 +386,14 @@ }, { "id": "curiox-ht2000", + "needs_hardware_testing": true, "vendor": "Curiox", "name": "HT2000", "kind": "plate washer", "capabilities": [ "plate washing" ], - "status": "mostly", + "status": "wip", "api": "pylabrobot.curiox.CurioxHT2000", "api_version": "v1", "code_slug": "curiox", @@ -553,6 +556,7 @@ }, { "id": "highres-ambistore", + "needs_hardware_testing": true, "vendor": "HighRes Biosolutions", "name": "AmbiStore", "kind": "storage", @@ -618,6 +622,7 @@ }, { "id": "highres-tundrastore", + "needs_hardware_testing": true, "vendor": "HighRes Biosolutions", "name": "TundraStore", "kind": "storage", @@ -705,9 +710,9 @@ "vendor": "Inheco", "name": "Thermoshake", "models": [ - {"name": "Thermoshake", "status": "full"}, - {"name": "Thermoshake AC", "status": "wip"}, - {"name": "Thermoshake RM", "status": "wip"} + {"name": "Thermoshake", "status": "wip", "needs_hardware_testing": true}, + {"name": "Thermoshake AC", "status": "wip", "needs_hardware_testing": true}, + {"name": "Thermoshake RM", "status": "wip", "needs_hardware_testing": false} ], "kind": "heater shaker", "capabilities": [ @@ -715,7 +720,7 @@ "active cooling", "shaking" ], - "status": "full", + "status": "wip", "api": "pylabrobot.inheco.inheco_thermoshake", "api_version": "v1", "code_slug": "inheco", @@ -726,13 +731,14 @@ }, { "id": "kbioscience-kube", + "needs_hardware_testing": true, "vendor": "KBioscience", "name": "KUBE", "kind": "sealer", "capabilities": [ "sealing" ], - "status": "mostly", + "status": "wip", "api": "pylabrobot.kbioscience.KBioscienceKUBE", "api_version": "v1", "code_slug": "kbioscience", @@ -745,9 +751,9 @@ "vendor": "KBiosystems", "name": "Ultraseal", "models": [ - {"name": "Ultraseal ePRO", "status": "mostly"}, - {"name": "Ultraseal PRO", "status": "mostly"}, - {"name": "Ultraseal XT PRO", "status": "mostly"} + {"name": "Ultraseal ePRO", "status": "wip", "needs_hardware_testing": true}, + {"name": "Ultraseal PRO", "status": "mostly", "needs_hardware_testing": false}, + {"name": "Ultraseal XT PRO", "status": "wip", "needs_hardware_testing": true} ], "kind": "sealer", "capabilities": [ @@ -794,6 +800,7 @@ }, { "id": "mettler-toledo-mt-sics", + "needs_hardware_testing": true, "vendor": "Mettler Toledo", "name": "MT-SICS scales and weigh modules", "models": [ @@ -1292,7 +1299,7 @@ {"name": "WXS205", "status": "wip"}, {"name": "WXS205DU", "status": "wip"}, {"name": "WXS205S/15", "status": "wip"}, - {"name": "WXS205SDU/15", "status": "full"}, + {"name": "WXS205SDU/15", "status": "full", "needs_hardware_testing": false}, {"name": "WXS205SDUV/15", "status": "wip"}, {"name": "WXS205SV/15", "status": "wip"}, {"name": "WXS26", "status": "wip"}, @@ -1844,13 +1851,14 @@ }, { "id": "sartorius-entris2", + "needs_hardware_testing": true, "vendor": "Sartorius", "name": "Entris II", "kind": "scale", "capabilities": [ "weighing" ], - "status": "mostly", + "status": "wip", "api": "pylabrobot.sartorius.SartoriusEntris2", "api_version": "v1", "code_slug": "sartorius", @@ -1903,18 +1911,19 @@ }, { "id": "thermo-fisher-alps", + "needs_hardware_testing": true, "vendor": "Thermo Fisher", "name": "ALPS", "models": [ - {"name": "ALPS 300", "status": "mostly"}, - {"name": "ALPS 3000", "status": "mostly"}, - {"name": "ALPS 5000", "status": "mostly"} + {"name": "ALPS 300", "status": "wip"}, + {"name": "ALPS 3000", "status": "wip"}, + {"name": "ALPS 5000", "status": "wip"} ], "kind": "sealer", "capabilities": [ "sealing" ], - "status": "mostly", + "status": "wip", "api": "pylabrobot.thermo_fisher.alps", "api_version": "v1", "code_slug": "thermo_fisher/alps", diff --git a/docs/contributor_guide/contributing.md b/docs/contributor_guide/contributing.md index 8ee0d9139b6..e8fdd8e9761 100644 --- a/docs/contributor_guide/contributing.md +++ b/docs/contributor_guide/contributing.md @@ -78,6 +78,11 @@ To build the documentation, run `make docs` in the root directory. The documenta ## Common Tasks +### Testing a device + +You can contribute by testing PyLabRobot on hardware you have access to and reporting successful +runs or failures. See {doc}`/user_guide/needs-testing` for the device table and reporting process. + ### Fixing a bug Bug fixes are an easy way to get started contributing. diff --git a/docs/contributor_guide/device-registry.md b/docs/contributor_guide/device-registry.md index 06b2c9386c5..1e2d48fc7c4 100644 --- a/docs/contributor_guide/device-registry.md +++ b/docs/contributor_guide/device-registry.md @@ -41,6 +41,7 @@ Append an object to `docs/_static/devices.json`: | `models` | no | Model objects when one entry covers several models. `name` is required; `status` may be `wip`, `basic`, `mostly`, or `full` and defaults to the device status when omitted. Models render as searchable sub-rows with their support status beneath the device. | | `kind` | yes | Device type, e.g. `plate reader`, `sealer`, `arm`. Must be one of `KINDS` in `docs/_exts/plr_devices/data.py`. | | `status` | yes | One of `wip`, `basic`, `mostly`, `full`. See {doc}`/user_guide/machines` for what each level means. | +| `needs_hardware_testing` | no | Boolean, default `false`. Set `true` when hardware testing is needed, based on the device's docs, driver warnings, or reports. Each model can also set this boolean; omitted model flags inherit the device flag. | | `capabilities` | no | Core functions, e.g. `["heating", "shaking"]`. Must come from `CAPABILITIES` in `docs/_exts/plr_devices/data.py`. These drive the badges and the capability filter. | | `api` | no | Import path of the driver class, e.g. `pylabrobot.curiox.CurioxHT2000`. | | `api_version` | no | `v1`, or `v0` for drivers still under `pylabrobot.legacy`. | @@ -117,6 +118,28 @@ Options narrow it down: `filters` take `false` to hide the search box or the chips, which is useful for a short, pre-filtered list on a vendor page. +The {doc}`/user_guide/needs-testing` page uses: + +````md +```{device-table} +:needs-hardware-testing: +``` +```` + +This includes devices that need hardware testing and families with at least one model that needs +it. Only flagged models appear in this table; other tables and cards keep the complete model list. +An explicit model flag overrides the device default. For example, a family can have +`"needs_hardware_testing": true` with `"needs_hardware_testing": false` on its tested model. +A family whose models are all explicitly `false` is excluded. An omitted or `false` flag means +there is no testing request recorded, not a claim that all firmware or operations are verified. + +Use existing docs, code warnings, or linked hardware reports to set the flag; do not infer it from +`wip` or any other support level. A device or model awaiting initial hardware verification has +`status: "wip"`, even if its shared driver works on other models. After reviewing a hardware report, +update only the tested model or device, preserve flags for untested siblings, and link the evidence +in the device guide. Follow +the reporting and follow-up process on {doc}`/user_guide/needs-testing`. + ## Rendering a card `device-card` renders one device. It works anywhere MyST is parsed, including markdown cells in the diff --git a/docs/user_guide/index.md b/docs/user_guide/index.md index 3bbce5af02e..11257347aa2 100644 --- a/docs/user_guide/index.md +++ b/docs/user_guide/index.md @@ -18,6 +18,7 @@ getting-started/units :hidden: machines +needs-testing definitions generic/index 00_liquid-handling/_liquid-handling diff --git a/docs/user_guide/machines.md b/docs/user_guide/machines.md index 627535bfb41..a8aee9dc69d 100644 --- a/docs/user_guide/machines.md +++ b/docs/user_guide/machines.md @@ -1,9 +1,12 @@ # Supported Machines -Every machine PyLabRobot supports, and how complete each driver is. Some are still work in +Every machine PyLabRobot supports, and its current support level. Some are still work in progress (WIP) — if you have one of those, or a machine that is not listed at all, get in touch on the [forum](https://discuss.pylabrobot.org). +Have access to hardware? See {doc}`needs-testing` for devices awaiting verification and how to +report what works or fails. + ```{device-table} ``` @@ -26,9 +29,10 @@ PyLabRobot does not solve that. **Type** is the one label that names what a mach filterable, so a machine you think of as a shaker is still findable by someone who thinks of it as a heater. -**Support** is how complete the PyLabRobot integration is: +**Support** is how complete the PyLabRobot integration is for the listed device or model. A shared +driver does not establish support for every model it can communicate with. -- **WIP** — work in progress. +- **WIP** — work in progress, including models awaiting initial hardware verification. - **Basic** — core functionality is available, integrated into `pylabrobot:main`. - **Mostly** — most capabilities are available, but some known commands are still missing. - **Full** — comprehensive support (≥90% of capabilities), with documentation. diff --git a/docs/user_guide/needs-testing.md b/docs/user_guide/needs-testing.md new file mode 100644 index 00000000000..42c43da388e --- /dev/null +++ b/docs/user_guide/needs-testing.md @@ -0,0 +1,57 @@ +# Devices that need testing + +Have access to one of these devices? Run your device's PyLabRobot hello-world example and report +what works and what fails. Reports of successful runs, partial success, and failures are all useful; +you do not need to write a driver or open a pull request to contribute. + +## Needs testing + +This table lists devices and models flagged as needing hardware testing based on their existing +documentation and driver warnings. **Support** applies to the listed device or model. A shared +driver may support one model while another still needs hardware testing. Models awaiting initial +hardware verification are marked **WIP**. + +Use **Show models**, expand a device row, or search for your exact model. Only models that need +testing appear here. Follow **docs** for the guide and known limitations, **code** for the driver, +or **Manager** to find the person looking after it. Some entries still need implementation work; +check their guide before attempting a run. + +```{device-table} +:needs-hardware-testing: +``` + +## Test and report + +1. Choose your exact model and read its guide, including setup instructions and known limitations. + If you need help, the listed manager is available on the + [forum](https://discuss.pylabrobot.org). +2. Record the model, firmware, connection settings, and PyLabRobot version or Git commit + (`git rev-parse HEAD` from your checkout). Start with the guide's hello-world example and test + only the operations appropriate for your hardware and setup. +3. Record each operation you tried, its expected result, and what actually happened on the + instrument. A command returning without an error is not enough to confirm the physical result. + Include the script or notebook, output, and relevant I/O logs. Mark steps you did not run as + **Not tested**; simulation and mock tests do not count as hardware verification. +4. [Submit a device test report](https://github.com/PyLabRobot/pylabrobot/issues/new?template=device-test-report.md). + Use **Works**, **Partly works**, or **Does not work** in the title and fill in the template. + Search existing issues first; if there is a report for the same model and problem, add your + results there. You can also post the same details on the + [forum](https://discuss.pylabrobot.org) and link any related issue or pull request. + +For example, if connection and status queries work but plate retrieval fails, report **Partly +works**, with separate rows for each operation. If setup fails, report **Does not work**, include +the error, and mark later steps as **Not tested**. Successful reports should also include the +code and observed results so someone else can reproduce them. + +## What happens to a report + +The device manager or a maintainer reviews the evidence and follows up on missing details or +failures. A report applies to the model, firmware, and operations you actually tested; it does +not verify every model in a family. + +Once the relevant hardware checks pass, a contributor or maintainer can open a pull request to +set `needs_hardware_testing` to `false` for that device or model in the +{doc}`/contributor_guide/device-registry`, link the report in the device's guide, and update any +not-tested warnings in the guide and driver to match the verified scope. Keep the flag set when +testing remains incomplete or a failure still needs a fix and another hardware run. Update the +tested device or model's support level to reflect its verified capabilities.