diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json
index 7989bcd..6fb6ff6 100644
--- a/.claude-plugin/marketplace.json
+++ b/.claude-plugin/marketplace.json
@@ -8,7 +8,7 @@
"plugins": [
{
"name": "plc-programming",
- "description": "Design, write, convert, review and debug PLC programs: IEC 61131-3 (ST, LD, FBD, SFC, IL), PLCopen XML / IEC 61131-10, Siemens TIA (SCL, SimaticML, SIMATIC SD), Schneider (Machine Expert, Control Expert), CODESYS, TwinCAT, ABB Freelance, Rockwell L5X/L5K, IEC 61499, AutomationML, DEXPI, PackML/ISA-88, OPC UA; with a stdlib-only structural validator and an edit-time validation hook.",
+ "description": "Design, write, convert, review and debug PLC programs: IEC 61131-3 (ST, LD, FBD, SFC, IL), PLCopen XML / IEC 61131-10, Siemens TIA (SCL, SimaticML, SIMATIC SD), Schneider (Machine Expert, Control Expert), CODESYS, TwinCAT, ABB Freelance, Rockwell L5X/L5K, IEC 61499, AutomationML, DEXPI, PackML/ISA-88, OPC UA; with a stdlib-only structural validator, an edit-time validation hook, and an offline reader for CODESYS / Machine Expert .project files.",
"source": "./"
}
]
diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json
index b9eeda7..c87a752 100644
--- a/.claude-plugin/plugin.json
+++ b/.claude-plugin/plugin.json
@@ -1,7 +1,7 @@
{
"$schema": "https://anthropic.com/claude-code/plugin.schema.json",
"name": "plc-programming",
- "version": "0.1.0",
+ "version": "0.2.0",
"description": "PLC programming across IEC 61131-3 (ST, LD, FBD, SFC, IL), PLCopen XML / IEC 61131-10, Siemens TIA (SCL, SimaticML, SIMATIC SD), Schneider (Machine Expert, Control Expert), CODESYS, TwinCAT, ABB Freelance, Rockwell L5X/L5K, IEC 61499, AutomationML, DEXPI, PackML/ISA-88 and OPC UA: design, write, convert, review and debug programs, with a stdlib-only structural validator.",
"author": {
"name": "SmailG",
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index b974066..57cc341 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -12,14 +12,15 @@ privately, as described in [SECURITY.md](SECURITY.md). Pull requests use the tem
| `agents/plc-reviewer.md` | Read-only reviewer used by `/plc-programming:review` |
| `hooks/hooks.json` | PostToolUse hook that runs the validator on every file Claude writes or edits |
| `scripts/plc_validate.py` | Validator CLI and hook entry point |
+| `scripts/codesys_project.py` | Offline reader for CODESYS 3.5 / Machine Expert `.project` files (`list`, `st`, `dump-st`, `graph`, `info`, `diff`, `xref`); `scripts/codesysproj/` holds `binary.py` (varint, string table), `project.py` (objects, kinds, text), `graph.py` (LD/FBD records) and `graphview.py` (their text and JSON), `export.py` (`dump-st`), `buildinfo.py` (`info`), `diff.py` (`diff`), `xref.py` and `stscan.py` (`xref`) |
| `scripts/plccheck/` | One module per format (`st.py`, `plcopen.py`, `simaticml.py`, `twincat.py`, `logix.py`, `schneider.py`, `iec61499.py`, `aml.py`, `dexpi.py`, `tabular.py`); `findings.py` holds the finding type, `xmlutil.py` the line-numbered XML parser |
-| `tests/` | `test_st.py`, `test_formats.py` (validator rules), `test_skills.py` (frontmatter, templates, every `iecst` block lints clean) |
+| `tests/` | `test_st.py`, `test_formats.py` (validator rules), `test_codesys_project.py` and `test_codesys_tools.py` (the `.project` reader and its `info`, `diff` and `xref`, on synthetic projects only), `test_skills.py` (frontmatter, templates, every `iecst` block lints clean) |
| `evals/` | `claude plugin eval` cases comparing the plugin against a no-plugin baseline |
## Rules
-- **The validator uses only the Python standard library** and must run on Python 3.9 (CI runs
- 3.9 and 3.12). Optional tools such as `xmllint` or `lxml` are allowed only behind `--xsd`.
+- **The validator and the `.project` reader use only the Python standard library** and must
+ run on Python 3.9 (CI runs 3.9 and 3.12). Optional tools such as `xmllint` or `lxml` are allowed only behind `--xsd`.
- **Every validator rule needs two tests:** an input it must catch and a valid control it must
accept. A rule tested only on bad input cannot show it is not firing on everything.
- **Every fact in a reference file is sourced or marked unverified.** Name the manual, standard
@@ -30,6 +31,8 @@ privately, as described in [SECURITY.md](SECURITY.md). Pull requests use the tem
- **Bump the version** in `.claude-plugin/plugin.json` on every change; `claude plugin update`
does nothing while it is unchanged.
- Do not bundle vendor schemas or manuals whose licence does not clearly permit redistribution.
+- **Test fixtures for the `.project` reader are synthetic.** Never commit a real `.project`,
+ or names, text or guids taken from one: it is someone's program.
## Tests
diff --git a/README.md b/README.md
index c7bd0d2..6fd7690 100644
--- a/README.md
+++ b/README.md
@@ -45,7 +45,7 @@ git clone https://github.com/SmailG/plc-programming.git .agents/plugins/plc-prog
| `schneider` skill | automatic | Machine Expert (M241/M251/M262), Control Expert (M340/M580), Automation Expert, Modbus, EtherNet/IP, OPC UA |
| `iec-61131-3` skill | automatic | Ed.2–Ed.4, the five languages, POUs, OOP, data types, the standard library, PLCopen guidelines, Motion and Safety |
| `exchange-formats` skill | automatic | PLCopen XML (TC6 v2.0/v2.01), IEC 61131-10, AutomationML (IEC 62714), DEXPI / ISO 15926 |
-| `other-platforms` skill | automatic | Beckhoff TwinCAT, CODESYS, ABB Freelance (PRT/CSV) / AC500 / 800xA, Rockwell L5X/L5K |
+| `other-platforms` skill | automatic | Beckhoff TwinCAT, CODESYS (incl. reading a binary `.project` offline: ST, build identity, version diff, cross-reference), ABB Freelance (PRT/CSV) / AC500 / 800xA, Rockwell L5X/L5K |
| `iec-61499` skill | automatic | event-driven FBs, ECC, 4diac file formats |
| `packml-isa88-opcua` skill | automatic | PackML states and PackTags, ISA-88 / IEC 61512, OPC UA (OPC 30000/30001/30050) |
| `/plc-programming:new-pou` | user only | scaffold a POU for a chosen platform and format |
@@ -78,6 +78,31 @@ The official XSDs (PLCopen TC6, IEC 61131-10) are **not bundled** because their
do not clearly permit redistribution. Download them from
and pass `--xsd`.
+## The CODESYS `.project` reader
+
+```bash
+python3 scripts/codesys_project.py list project.project
+python3 scripts/codesys_project.py st project.project
+python3 scripts/codesys_project.py dump-st project.project out/
+python3 scripts/codesys_project.py graph project.project [--json]
+python3 scripts/codesys_project.py info project.project
+python3 scripts/codesys_project.py diff old.project new.project []
+python3 scripts/codesys_project.py xref project.project
+```
+
+Reads a CODESYS 3.5 / EcoStruxure Machine Expert `.project` without the IDE (standard
+library only, Python 3.9+). **The format is undocumented**: the reader is reverse-engineered
+and was checked on Machine Expert V2.x files only. ST text and the object inventory are
+decoded (`dump-st` writes every ST POU and every GVL and DUT declaration); graphical (LD/FBD)
+records are listed with every interpretation marked as a heuristic, and the wiring is not
+decoded. `info` prints each application's CodeId / DataId (a lead to which saved project the
+controller runs; the IDE's comparison decides) and the last save time; `diff` compares two
+versions of a POU or of the whole project with ids and offsets left out; `xref` lists every use
+of a name, by name only (heuristic). Damaged, encrypted or oversized zip entries are reported,
+not crashed on. Prefer a PLCopen XML export from the IDE when you
+can get one, and verify graphical logic in the IDE. Details:
+[codesys-project-files.md](skills/other-platforms/references/codesys-project-files.md).
+
## Tests and evals
```bash
@@ -86,7 +111,8 @@ claude plugin eval . --trust-plugin --allow-tools Write Edit "Bash(python3 *)" -
```
- **Unit tests:** every validator rule is exercised by an input it must catch and by a
- valid control it must pass. Every shipped template and every `iecst` code block in the
+ valid control it must pass. The `.project` reader is tested on synthetic projects built in
+ the test, never on a real program. Every shipped template and every `iecst` code block in the
skills must lint clean.
- **Evals:** they compare the plugin against a no-plugin baseline on edition facts, SFC
semantics, a classic scan-cycle bug, PLCopen XML generation, SimaticML namespaces,
diff --git a/plugin.json b/plugin.json
index 257faa5..bfedbe6 100644
--- a/plugin.json
+++ b/plugin.json
@@ -1,7 +1,7 @@
{
"name": "plc-programming",
"displayName": "PLC Programming",
- "version": "0.1.0",
+ "version": "0.2.0",
"description": "PLC programming across IEC 61131-3 (ST, LD, FBD, SFC, IL), PLCopen XML / IEC 61131-10, Siemens TIA, Schneider, CODESYS, TwinCAT, ABB Freelance, Rockwell L5X/L5K, IEC 61499, AutomationML, DEXPI, PackML/ISA-88 and OPC UA with a stdlib-only structural validator.",
"author": {
"name": "SmailG",
diff --git a/scripts/codesys_project.py b/scripts/codesys_project.py
new file mode 100644
index 0000000..7ae0bbd
--- /dev/null
+++ b/scripts/codesys_project.py
@@ -0,0 +1,249 @@
+#!/usr/bin/env python3
+"""Read a CODESYS 3.5 / EcoStruxure Machine Expert `.project` file offline (no IDE).
+
+ codesys_project.py list PROJECT [--json] [--kind KIND]
+ codesys_project.py st PROJECT NAME_OR_GUID [--decl | --impl]
+ codesys_project.py dump-st PROJECT OUTDIR
+ codesys_project.py graph PROJECT NAME_OR_GUID [--json]
+ codesys_project.py info PROJECT [--json]
+ codesys_project.py diff OLD NEW [NAME_OR_GUID] [--json]
+ codesys_project.py xref PROJECT NAME [--json]
+
+The `.project` format is undocumented; this reader is reverse-engineered and was checked
+on Machine Expert V2.x projects only. ST text and the object inventory are decoded from
+the file's own structure. Graphical wiring is NOT decoded: `graph` lists element records
+and ids, and every interpretation is marked as a heuristic. Prefer a PLCopen XML export
+from the IDE when you can get one, and confirm graphical logic in the IDE before acting
+on it. See skills/other-platforms/references/codesys-project-files.md.
+
+`info` prints each application's CodeId / DataId (the build identity) and the project's last
+save time. `diff` compares two versions: with a POU it prints the unified diff of its text or
+of its decoded graph with ids and offsets left out; without one it lists the objects added,
+removed and changed. `xref` lists every ST and LD/FBD use of a name, by name only (heuristic).
+
+Exit: 0 clean (a difference between two projects is not a problem), 1 result printed but some
+objects had problems, 2 usage or unreadable input.
+Standard library only; Python 3.9 or later.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+from typing import List, Optional, Tuple
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+
+from codesysproj import buildinfo, diff, xref # noqa: E402
+from codesysproj.binary import FormatError # noqa: E402
+from codesysproj.export import dump_st # noqa: E402
+from codesysproj.graph import decode_graph # noqa: E402
+from codesysproj.graphview import graph_to_dict, render_graph_text # noqa: E402
+from codesysproj.project import ( # noqa: E402
+ KIND_DUT, KIND_GRAPHICAL, KIND_GVL, KIND_ST, KINDS, ObjectRecord, Project, find_objects, load_project,
+)
+
+EXIT_OK, EXIT_PROBLEMS, EXIT_FATAL = 0, 1, 2
+
+
+def record_dict(rec: ObjectRecord) -> dict:
+ return {
+ "guid": rec.guid, "name": rec.name, "kind": rec.kind, "subtype": rec.subtype,
+ "language": rec.language, "path": rec.path, "device": rec.device,
+ "application": rec.application, "parent_guid": rec.parent_guid, "problems": rec.problems,
+ }
+
+
+def format_list(records: List[ObjectRecord]) -> str:
+ lines = ["# guid kind(subtype) name ancestors (kind and subtype are heuristics)"]
+ for rec in records:
+ where = "/".join(rec.path) if rec.path else "-"
+ kind = rec.kind + ("(%s)" % rec.subtype if rec.subtype else "")
+ lines.append("%s %-26s %s %s" % (rec.guid, kind, rec.name if rec.name is not None else "?", where))
+ return "\n".join(lines)
+
+
+def resolve_one(project: Project, query: str) -> ObjectRecord:
+ """The single object a NAME_OR_GUID names; FormatError if none or several."""
+ matches = find_objects(project, query)
+ if not matches:
+ raise FormatError("no object with guid, guid prefix or name %r" % query)
+ if len(matches) > 1:
+ lines = ["%r is ambiguous (%d objects); give a guid or a guid prefix:" % (query, len(matches))]
+ lines += [" %s %s %s" % (r.guid, r.kind, "/".join(r.path + [r.name or "?"])) for r in matches]
+ raise FormatError("\n".join(lines))
+ return matches[0]
+
+
+def report_problems(records: List[ObjectRecord], label: str = "") -> int:
+ """Print each object's problems to stderr; return how many objects had any."""
+ bad = [r for r in records if r.problems]
+ prefix = "PROBLEM %s" % label if label else "PROBLEM"
+ for rec in bad:
+ for problem in rec.problems:
+ print("%s %s (%s): %s" % (prefix, rec.guid, rec.name or "?", problem), file=sys.stderr)
+ return len(bad)
+
+
+def cmd_list(project: Project, args: argparse.Namespace) -> int:
+ records = [r for r in project.records() if args.kind in (None, r.kind)]
+ if args.json:
+ print(json.dumps([record_dict(r) for r in records], indent=2, ensure_ascii=False))
+ else:
+ print(format_list(records))
+ return EXIT_PROBLEMS if report_problems(records) else EXIT_OK
+
+
+def cmd_st(project: Project, args: argparse.Namespace) -> int:
+ rec = resolve_one(project, args.target)
+ if rec.kind not in (KIND_ST, KIND_GRAPHICAL, KIND_GVL, KIND_DUT) or rec.declaration is None:
+ raise FormatError("%s (%s) is kind %r: there is no ST text to print" % (rec.guid, rec.name, rec.kind))
+ if args.decl:
+ sys.stdout.write(rec.declaration)
+ elif args.impl:
+ if rec.kind == KIND_GRAPHICAL:
+ raise FormatError("%s (%s) is graphical: its implementation is not ST; use `graph`" % (rec.guid, rec.name))
+ if rec.kind != KIND_ST:
+ raise FormatError("%s (%s) is a %s: it has a declaration only" % (rec.guid, rec.name, rec.kind))
+ sys.stdout.write(rec.implementation or "")
+ else:
+ print("(* declaration: %s %s *)" % (rec.name, rec.guid))
+ print(rec.declaration)
+ if rec.kind == KIND_ST:
+ print("(* implementation *)")
+ print(rec.implementation or "")
+ elif rec.kind == KIND_GRAPHICAL:
+ print("(* implementation: graphical (%s), not ST; use `graph` *)" % rec.language)
+ return EXIT_PROBLEMS if report_problems([rec]) else EXIT_OK
+
+
+def cmd_dump_st(project: Project, args: argparse.Namespace) -> int:
+ written = dump_st(project, args.outdir)
+ for rec, files in written:
+ print("%s %s" % (rec.guid, " ".join(files)))
+ print("wrote %d object(s) to %s" % (len(written), args.outdir))
+ return EXIT_PROBLEMS if report_problems([r for r, _ in written]) else EXIT_OK
+
+
+def cmd_graph(project: Project, args: argparse.Namespace) -> int:
+ rec = resolve_one(project, args.target)
+ if rec.kind != KIND_GRAPHICAL:
+ raise FormatError("%s (%s) is kind %r, not graphical" % (rec.guid, rec.name, rec.kind))
+ result = decode_graph(rec, project.blobs[rec.guid], project.strings)
+ print(json.dumps(graph_to_dict(result), indent=2, ensure_ascii=False) if args.json else render_graph_text(result))
+ return EXIT_PROBLEMS if (report_problems([rec]) or result.notes) else EXIT_OK
+
+
+def cmd_info(args: argparse.Namespace) -> int:
+ info = buildinfo.read_info(args.project)
+ print(json.dumps(buildinfo.info_to_dict(info), indent=2, ensure_ascii=False) if args.json
+ else buildinfo.render_info_text(info))
+ for note in info.notes:
+ print("NOTE %s" % note, file=sys.stderr)
+ return EXIT_PROBLEMS if info.notes else EXIT_OK
+
+
+def resolve_pair(old: Project, new: Project, query: str) -> Tuple[ObjectRecord, ObjectRecord]:
+ """The object `query` names, in both versions. The name may be the old or the new one (a rename):
+ it is resolved where it exists and the counterpart is found by guid."""
+ found = [(label, p) for label, p in (("NEW", new), ("OLD", old)) if find_objects(p, query)]
+ if not found:
+ raise FormatError("%r not found in OLD (%s) or NEW (%s)" % (query, old.path, new.path))
+ rec = resolve_one(found[0][1], query)
+ other_label, other = ("OLD", old) if found[0][0] == "NEW" else ("NEW", new)
+ if rec.guid not in other.objects:
+ raise FormatError("%s (%s) is not in %s (%s): it was %s"
+ % (rec.guid, rec.name, other_label, other.path, "added" if other is old else "removed"))
+ pair = (other.objects[rec.guid], rec) if other is old else (rec, other.objects[rec.guid])
+ for label, r in zip(("OLD", "NEW"), pair):
+ if r.kind not in diff.COMPARABLE_KINDS:
+ raise FormatError("%s (%s) is kind %r in %s: there is nothing to compare" % (r.guid, r.name, r.kind, label))
+ return pair
+
+
+def report_notes(notes: List[str]) -> int:
+ for note in notes:
+ print("NOTE %s" % note, file=sys.stderr)
+ return len(notes)
+
+
+def cmd_diff(args: argparse.Namespace) -> int:
+ old, new = load_project(args.old), load_project(args.new)
+ if args.target is None:
+ result = diff.project_diff(old, new)
+ print(json.dumps(diff.project_diff_to_dict(result), indent=2, ensure_ascii=False) if args.json
+ else diff.render_project_diff(result))
+ problems = report_problems(old.records(), "OLD") + report_problems(new.records(), "NEW")
+ return EXIT_PROBLEMS if report_notes(result.notes) or problems else EXIT_OK
+ a, b = resolve_pair(old, new, args.target)
+ result = diff.object_diff(old, a, new, b)
+ print(json.dumps(diff.object_diff_to_dict(result), indent=2, ensure_ascii=False) if args.json
+ else diff.render_object_diff(result))
+ problems = report_problems([a], "OLD") + report_problems([b], "NEW")
+ return EXIT_PROBLEMS if report_notes(result.notes) or problems else EXIT_OK
+
+
+def cmd_xref(project: Project, args: argparse.Namespace) -> int:
+ result = xref.find_references(project, args.name)
+ print(json.dumps(xref.xref_to_dict(result), indent=2, ensure_ascii=False) if args.json
+ else xref.render_xref_text(result))
+ return EXIT_PROBLEMS if report_problems(project.records()) or result.notes else EXIT_OK
+
+
+def build_parser() -> argparse.ArgumentParser:
+ parser = argparse.ArgumentParser(prog="codesys_project.py", description=__doc__.split("\n\n")[0])
+ sub = parser.add_subparsers(dest="command", required=True)
+ p_list = sub.add_parser("list", help="every object: guid, kind, name, ancestors")
+ p_list.add_argument("project")
+ p_list.add_argument("--json", action="store_true")
+ p_list.add_argument("--kind", choices=KINDS, help="only objects of this (heuristic) kind")
+ p_st = sub.add_parser("st", help="print a POU's declaration and implementation text (a GVL or DUT: its declaration)")
+ p_st.add_argument("project")
+ p_st.add_argument("target", help="object name, guid or unique guid prefix (8+ characters)")
+ part = p_st.add_mutually_exclusive_group()
+ part.add_argument("--decl", action="store_true", help="print only the declaration, raw")
+ part.add_argument("--impl", action="store_true", help="print only the implementation, raw")
+ p_dump = sub.add_parser("dump-st", help="write every ST POU as _.decl.st / .impl.st, GVLs and DUTs as .decl.st")
+ p_dump.add_argument("project")
+ p_dump.add_argument("outdir")
+ p_graph = sub.add_parser("graph", help="element records and network comments of an LD/FBD POU")
+ p_graph.add_argument("project")
+ p_graph.add_argument("target", help="object name, guid or unique guid prefix (8+ characters)")
+ p_graph.add_argument("--json", action="store_true")
+ p_info = sub.add_parser("info", help="CodeId / DataId per device application, and the last save time")
+ p_info.add_argument("project")
+ p_info.add_argument("--json", action="store_true")
+ p_diff = sub.add_parser("diff", help="what changed between two versions: one POU, or every object")
+ p_diff.add_argument("old")
+ p_diff.add_argument("new")
+ p_diff.add_argument("target", nargs="?", help="object name, guid or unique guid prefix; omit to list every change")
+ p_diff.add_argument("--json", action="store_true")
+ p_xref = sub.add_parser("xref", help="every ST and LD/FBD use of a variable, instance or type name (heuristic)")
+ p_xref.add_argument("project")
+ p_xref.add_argument("name", help="a name or dotted path such as Gvl.Counter; case-insensitive")
+ p_xref.add_argument("--json", action="store_true")
+ return parser
+
+
+COMMANDS = {"list": cmd_list, "st": cmd_st, "dump-st": cmd_dump_st, "graph": cmd_graph, "xref": cmd_xref}
+STANDALONE = {"info": cmd_info, "diff": cmd_diff} # load their own project(s)
+
+
+def main(argv: Optional[List[str]] = None) -> int:
+ args = build_parser().parse_args(argv)
+ for stream in (sys.stdout, sys.stderr): # a Windows console code page must not crash on a comment's č or đ
+ if hasattr(stream, "reconfigure"):
+ stream.reconfigure(errors="backslashreplace")
+ try:
+ if args.command in STANDALONE:
+ return STANDALONE[args.command](args)
+ return COMMANDS[args.command](load_project(args.project), args)
+ except FormatError as exc:
+ print("error: %s" % exc, file=sys.stderr)
+ return EXIT_FATAL
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/scripts/codesysproj/__init__.py b/scripts/codesysproj/__init__.py
new file mode 100644
index 0000000..c3a954a
--- /dev/null
+++ b/scripts/codesysproj/__init__.py
@@ -0,0 +1,6 @@
+"""Offline reader for CODESYS 3.5 / EcoStruxure Machine Expert `.project` files.
+
+The `.project` format is undocumented. Everything here is reverse-engineered from real
+projects; the notes are in skills/other-platforms/references/codesys-project-files.md.
+Standard library only; runs on Python 3.9.
+"""
diff --git a/scripts/codesysproj/binary.py b/scripts/codesysproj/binary.py
new file mode 100644
index 0000000..ec195cd
--- /dev/null
+++ b/scripts/codesysproj/binary.py
@@ -0,0 +1,54 @@
+"""Low-level primitives: the varint, the shared string table.
+
+Observed layout (reverse-engineered, not documented by the vendor):
+
+* a varint is LEB128: 7 data bits per byte, low group first, high bit = "more follows";
+* `__shared_data_storage_string_table__.auxiliary` is `count`, then `count` entries of
+ (index, byte length, UTF-8 bytes), all three as varints.
+"""
+
+from __future__ import annotations
+
+from typing import Dict, Tuple
+
+MAX_VARINT_BYTES = 10 # 70 bits: far past any index or length seen in a real project
+
+
+class FormatError(ValueError):
+ """The bytes do not follow the layout this reader understands."""
+
+
+def read_varint(buf: bytes, pos: int) -> Tuple[int, int]:
+ """Return (value, position after it). Raises FormatError on EOF or an overlong varint."""
+ value = 0
+ shift = 0
+ start = pos
+ while True:
+ if pos >= len(buf):
+ raise FormatError("varint at offset %d runs past the end of the data" % start)
+ if pos - start >= MAX_VARINT_BYTES:
+ raise FormatError("varint at offset %d is longer than %d bytes" % (start, MAX_VARINT_BYTES))
+ byte = buf[pos]
+ pos += 1
+ value |= (byte & 0x7F) << shift
+ shift += 7
+ if not byte & 0x80:
+ return value, pos
+
+
+def parse_string_table(data: bytes) -> Dict[int, str]:
+ """Parse the shared string table into {index: text}."""
+ declared, pos = read_varint(data, 0)
+ table: Dict[int, str] = {}
+ while pos < len(data):
+ index, pos = read_varint(data, pos)
+ length, pos = read_varint(data, pos)
+ if pos + length > len(data):
+ raise FormatError("string table entry %d (%d bytes) runs past the end of the file" % (index, length))
+ if index in table:
+ raise FormatError("string table index %d appears twice" % index)
+ table[index] = data[pos:pos + length].decode("utf-8", "replace")
+ pos += length
+ if len(table) != declared:
+ raise FormatError("string table declares %d entries but holds %d" % (declared, len(table)))
+ return table
diff --git a/scripts/codesysproj/buildinfo.py b/scripts/codesysproj/buildinfo.py
new file mode 100644
index 0000000..7fa2094
--- /dev/null
+++ b/scripts/codesysproj/buildinfo.py
@@ -0,0 +1,213 @@
+"""Build identity of a `.project`: the CodeId / DataId per device application, and the save stamp.
+
+Observed layout (reverse-engineered; two XML entries of the project zip, both plain text):
+
+* `devicesmetainformation.auxiliary` is `` holding one `` per controller. A
+ device has `` children (Name, DeviceType, DeviceId, DeviceVersion,
+ DeviceName) and one `` per application, itself holding Name, CodeId and DataId
+ as `System.Guid` data. The IDE compares CodeId and DataId with the controller to tell whether
+ the project matches what is downloaded (**not verified here**: the vendor does not document it).
+* `engineeringtoolsversions.auxiliary` is `` with a `` of ``
+ elements: CoreVersion, OpenCounter, SaveCounter, FirstTimeSaved, LastTimeSaved. The newest
+ LastTimeSaved is the project's last SAVE, not a compile time.
+
+There is no compile time in the project. The IDE writes it to the `.compileinfo` / `.bootinfo`
+files next to the project, which are not part of the zip.
+"""
+
+from __future__ import annotations
+
+import re
+import xml.etree.ElementTree as ET
+import xml.parsers.expat
+from dataclasses import dataclass, field
+from typing import List, Optional
+
+from .binary import FormatError
+from .zipread import entry_name, open_archive, read_entry
+
+DEVICES_ENTRY = "devicesmetainformation.auxiliary"
+HISTORY_ENTRY = "engineeringtoolsversions.auxiliary"
+_STAMP_RE = re.compile(r"^(\d{4})-(\d\d)-(\d\d)-(\d\d)-(\d\d)-(\d\d)$")
+_GUID_HEAD_RE = re.compile(r"^([0-9a-fA-F]{8})-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$")
+
+
+@dataclass
+class ApplicationInfo:
+ name: Optional[str]
+ code_id: Optional[str]
+ data_id: Optional[str]
+
+
+@dataclass
+class DeviceInfo:
+ name: Optional[str]
+ device_type: Optional[str]
+ device_id: Optional[str]
+ device_version: Optional[str]
+ device_name: Optional[str]
+ applications: List[ApplicationInfo] = field(default_factory=list)
+
+
+@dataclass
+class ProjectInfo:
+ devices: List[DeviceInfo] = field(default_factory=list)
+ product: Optional[str] = None
+ project_version: Optional[str] = None
+ last_saved: Optional[str] = None # "YYYY-MM-DD HH:MM:SS", the saving machine's clock
+ save_counter: Optional[int] = None
+ core_version: Optional[str] = None
+ notes: List[str] = field(default_factory=list)
+
+
+def short_id(guid: Optional[str]) -> Optional[str]:
+ """The first eight hex digits of a guid, which is what the rest of the guid (zeros) leaves."""
+ match = _GUID_HEAD_RE.match(guid or "")
+ return match.group(1).lower() if match else None
+
+
+def swapped_id(guid: Optional[str]) -> Optional[str]:
+ """The first four bytes reversed. A boot image stores the id this way; which order the IDE
+ shows is not verified, so both are printed."""
+ head = short_id(guid)
+ return "".join(reversed([head[i:i + 2] for i in range(0, 8, 2)])) if head else None
+
+
+def _parse_xml(data: bytes, entry: str, notes: List[str]) -> Optional[ET.Element]:
+ """The XML root, or None with a note. DOCTYPE and ENTITY declarations are refused, not expanded,
+ whatever the document's encoding: expat reports them before ElementTree sees any entity."""
+ if _has_doctype(data):
+ notes.append("%s: has a DOCTYPE or ENTITY declaration; not parsed" % entry)
+ return None
+ try:
+ return ET.fromstring(data)
+ except ET.ParseError as exc:
+ notes.append("%s: not well-formed XML (%s)" % (entry, exc))
+ return None
+
+
+class _Doctype(Exception):
+ pass
+
+
+def _has_doctype(data: bytes) -> bool:
+ def refuse(*_args) -> None:
+ raise _Doctype()
+ parser = xml.parsers.expat.ParserCreate()
+ parser.StartDoctypeDeclHandler = refuse
+ parser.EntityDeclHandler = refuse
+ try:
+ parser.Parse(data, True)
+ except _Doctype:
+ return True
+ except xml.parsers.expat.ExpatError:
+ return False # not well-formed: ElementTree reports it
+ return False
+
+
+def _data_values(node: ET.Element) -> dict:
+ return {d.get("Key"): (d.text or "").strip() for d in node.findall("Data")}
+
+
+def _devices(root: ET.Element) -> List[DeviceInfo]:
+ devices: List[DeviceInfo] = []
+ for node in root.findall("Device"):
+ v = _data_values(node)
+ device = DeviceInfo(v.get("Name"), v.get("DeviceType"), v.get("DeviceId"), v.get("DeviceVersion"),
+ v.get("DeviceName"))
+ for app in node.findall("Application"):
+ a = _data_values(app)
+ device.applications.append(ApplicationInfo(a.get("Name"), a.get("CodeId"), a.get("DataId")))
+ devices.append(device)
+ return devices
+
+
+def _newest_save(root: ET.Element, info: ProjectInfo) -> None:
+ best = None
+ for entry in root.iter("HistoryEntry"):
+ match = _STAMP_RE.match(entry.get("LastTimeSaved", ""))
+ if match and (best is None or match.groups() > best[0]):
+ best = (match.groups(), entry)
+ info.product = root.get("InformationalProductName")
+ info.project_version = root.get("EngineeringProjectVersion")
+ if best is not None:
+ g, entry = best
+ info.last_saved = "%s-%s-%s %s:%s:%s" % g
+ info.core_version = entry.get("CoreVersion")
+ counter = entry.get("SaveCounter", "")
+ info.save_counter = int(counter) if counter.isascii() and counter.isdigit() else None
+
+
+def _read_xml(archive, zinfo, entry: str, notes: List[str]) -> Optional[ET.Element]:
+ try:
+ data = read_entry(archive, zinfo)
+ except FormatError as exc:
+ notes.append("%s: %s" % (entry, exc))
+ return None
+ return _parse_xml(data, entry, notes)
+
+
+def read_info(path: str) -> ProjectInfo:
+ """Read the build identity. A missing or unreadable entry is a note; a non-zip raises FormatError."""
+ archive = open_archive(path)
+ info = ProjectInfo()
+ with archive:
+ entries = {entry_name(i): i for i in archive.infolist()}
+ if DEVICES_ENTRY not in entries:
+ info.notes.append("no %s entry: no build identity in this project" % DEVICES_ENTRY)
+ else:
+ root = _read_xml(archive, entries[DEVICES_ENTRY], DEVICES_ENTRY, info.notes)
+ if root is not None:
+ info.devices = _devices(root)
+ if not info.devices:
+ info.notes.append("%s: no element under <%s>" % (DEVICES_ENTRY, root.tag))
+ if HISTORY_ENTRY in entries:
+ root = _read_xml(archive, entries[HISTORY_ENTRY], HISTORY_ENTRY, info.notes)
+ if root is not None:
+ _newest_save(root, info)
+ return info
+
+
+def info_to_dict(info: ProjectInfo) -> dict:
+ return {
+ "devices": [
+ {"name": d.name, "device_type": d.device_type, "device_id": d.device_id,
+ "device_version": d.device_version, "device_name": d.device_name,
+ "applications": [
+ {"name": a.name, "code_id": a.code_id, "code_id_short": short_id(a.code_id),
+ "code_id_byte_swapped": swapped_id(a.code_id), "data_id": a.data_id,
+ "data_id_short": short_id(a.data_id), "data_id_byte_swapped": swapped_id(a.data_id)}
+ for a in d.applications]}
+ for d in info.devices],
+ "product": info.product, "project_version": info.project_version, "last_saved": info.last_saved,
+ "save_counter": info.save_counter, "core_version": info.core_version, "notes": info.notes,
+ }
+
+
+def _id_text(label: str, guid: Optional[str]) -> str:
+ if not guid:
+ return "%s -" % label
+ swapped = swapped_id(guid)
+ return "%s %s" % (label, guid) + (" (byte-swapped %s)" % swapped if swapped else "")
+
+
+def render_info_text(info: ProjectInfo) -> str:
+ lines = [
+ "# build identity, decoded from devicesmetainformation.auxiliary. The IDE compares CodeId and DataId",
+ "# with the controller (not verified here); which byte order it shows is not verified either, so both",
+ "# are printed. The project holds no compile time: that is in the .compileinfo beside the project file.",
+ ]
+ for device in info.devices:
+ lines.append("device %s type=%s id=%s version=%s name=%s" % (
+ device.name, device.device_type, device.device_id, device.device_version, device.device_name))
+ if not device.applications:
+ lines.append(" (no application)")
+ for app in device.applications:
+ lines.append(" application %s %s %s" % (app.name, _id_text("CodeId", app.code_id),
+ _id_text("DataId", app.data_id)))
+ if info.last_saved:
+ lines.append("last saved: %s (the save, not a compile; the saving machine's clock) save counter %s core %s %s %s"
+ % (info.last_saved, info.save_counter, info.core_version, info.product or "", info.project_version or ""))
+ else:
+ lines.append("last saved: not available")
+ return "\n".join(lines)
diff --git a/scripts/codesysproj/diff.py b/scripts/codesysproj/diff.py
new file mode 100644
index 0000000..cf01968
--- /dev/null
+++ b/scripts/codesysproj/diff.py
@@ -0,0 +1,191 @@
+"""Compare objects between two versions of a project.
+
+What is compared, per object kind: ST has a declaration and an implementation (text, line
+by line, line endings ignored); a graphical POU has a declaration (text) and a `graph`, the
+decoded networks rendered with everything that changes on a re-save left out: byte offsets,
+element ids, box header ids, empty-slot ids and raw bytes. Names, types, constants, modifier
+bytes, network comments, box types, instances, box nesting and the coil count stay in, so a
+constant that changes from 500 to 600 shows.
+
+Not compared, and therefore invisible here: the wiring (the graph decoder does not decode it),
+and bytes the graph decoder does not recognise. "No differences" means no difference in the
+fields above, not that the two objects are identical.
+"""
+
+from __future__ import annotations
+
+import difflib
+from dataclasses import dataclass, field
+from typing import Dict, List, Optional
+
+from .graph import decode_graph
+from .graphview import item_label
+from .project import KIND_DUT, KIND_GRAPHICAL, KIND_GVL, KIND_ST, ObjectRecord, Project
+
+COMPARABLE_KINDS = (KIND_ST, KIND_GRAPHICAL, KIND_GVL, KIND_DUT)
+CONTEXT_LINES = 2
+LIMITS_NOTE = ("# not compared: wiring (not decoded) and bytes the graph decoder does not recognise; "
+ "an empty diff does not mean the objects are identical")
+
+
+@dataclass
+class SectionDiff:
+ label: str # declaration | implementation | graph
+ lines: List[str] # unified diff, header lines first
+
+
+@dataclass
+class ObjectDiff:
+ old: ObjectRecord
+ new: ObjectRecord
+ sections: List[SectionDiff] = field(default_factory=list)
+ notes: List[str] = field(default_factory=list) # graph decode notes, both versions
+
+
+@dataclass
+class Change:
+ guid: str
+ name: Optional[str]
+ kind: str
+ path: List[str]
+ labels: List[str] = field(default_factory=list) # the sections that differ
+ moves: List[str] = field(default_factory=list) # "renamed from 'X'", "moved from 'a/b'"
+
+
+@dataclass
+class ProjectDiff:
+ added: List[ObjectRecord] = field(default_factory=list)
+ removed: List[ObjectRecord] = field(default_factory=list)
+ changed: List[Change] = field(default_factory=list)
+ unchanged: int = 0
+ not_compared: int = 0
+ notes: List[str] = field(default_factory=list)
+
+
+def graph_lines(project: Project, rec: ObjectRecord, notes: Optional[List[str]] = None) -> List[str]:
+ """The decoded graph as comparable lines: one per network comment and one per item."""
+ result = decode_graph(rec, project.blobs[rec.guid], project.strings)
+ if notes is not None:
+ notes.extend("%s: %s" % (rec.name, n) for n in result.notes)
+ lines: List[str] = []
+ for net in result.networks:
+ lines.append("network comment=%s" % ("none" if net.comment is None else repr(net.comment)))
+ lines += [item_label(item, stable=True) for item in net.items]
+ return lines
+
+
+def _text_lines(text: Optional[str]) -> List[str]:
+ return text.splitlines() if text else []
+
+
+def sections(project: Project, rec: ObjectRecord, notes: Optional[List[str]] = None) -> Dict[str, List[str]]:
+ """Comparable parts of an object, by label."""
+ parts: Dict[str, List[str]] = {"declaration": _text_lines(rec.declaration)}
+ if rec.kind == KIND_ST:
+ parts["implementation"] = _text_lines(rec.implementation)
+ if rec.kind == KIND_GRAPHICAL:
+ parts["graph"] = graph_lines(project, rec, notes)
+ return parts
+
+
+def _unified(label: str, old_name: Optional[str], new_name: Optional[str], old: List[str], new: List[str]) -> List[str]:
+ return list(difflib.unified_diff(old, new, "OLD %s %s" % (old_name, label), "NEW %s %s" % (new_name, label),
+ lineterm="", n=CONTEXT_LINES))
+
+
+def differing_labels(old: Dict[str, List[str]], new: Dict[str, List[str]]) -> List[str]:
+ order = ("declaration", "implementation", "graph")
+ return [lbl for lbl in order if (lbl in old or lbl in new) and old.get(lbl, []) != new.get(lbl, [])]
+
+
+def object_diff(old_project: Project, old: ObjectRecord, new_project: Project, new: ObjectRecord) -> ObjectDiff:
+ result = ObjectDiff(old, new)
+ old_parts = sections(old_project, old, result.notes)
+ new_parts = sections(new_project, new, result.notes)
+ for label in differing_labels(old_parts, new_parts):
+ result.sections.append(SectionDiff(label, _unified(label, old.name, new.name, old_parts.get(label, []),
+ new_parts.get(label, []))))
+ return result
+
+
+def _is_comparable(rec: ObjectRecord) -> bool:
+ return rec.kind in COMPARABLE_KINDS
+
+
+def project_diff(old: Project, new: Project) -> ProjectDiff:
+ """Objects matched by guid (a guid survives a save; the names are what people change)."""
+ result = ProjectDiff()
+ result.added = [new.objects[g] for g in sorted(set(new.objects) - set(old.objects))]
+ result.removed = [old.objects[g] for g in sorted(set(old.objects) - set(new.objects))]
+ for guid in sorted(set(old.objects) & set(new.objects)):
+ a, b = old.objects[guid], new.objects[guid]
+ if not (_is_comparable(a) and _is_comparable(b)):
+ result.not_compared += 1
+ continue
+ labels = differing_labels(sections(old, a, result.notes), sections(new, b, result.notes))
+ moves = []
+ if a.name != b.name:
+ moves.append("renamed from %r" % a.name)
+ if a.path != b.path:
+ moves.append("moved from %r" % "/".join(a.path))
+ if labels or moves:
+ result.changed.append(Change(guid, b.name, b.kind, b.path, labels, moves))
+ else:
+ result.unchanged += 1
+ return result
+
+
+def _where(name: Optional[str], path: List[str]) -> str:
+ return "/".join(path + [name or "?"])
+
+
+def render_object_diff(result: ObjectDiff) -> str:
+ lines: List[str] = []
+ if result.old.kind != result.new.kind:
+ lines.append("# kind changed: %s -> %s" % (result.old.kind, result.new.kind))
+ if result.old.name != result.new.name:
+ lines.append("# renamed: %r -> %r" % (result.old.name, result.new.name))
+ if KIND_GRAPHICAL in (result.old.kind, result.new.kind):
+ lines.append(LIMITS_NOTE)
+ if not result.sections:
+ lines.append("no differences in the compared fields (%s)" % _where(result.new.name, result.new.path))
+ for section in result.sections:
+ lines += section.lines
+ return "\n".join(lines)
+
+
+def render_project_diff(result: ProjectDiff) -> str:
+ lines = ["# objects matched by guid; ST text and decoded LD/FBD graphs compared, other kinds not. "
+ "Give a POU name to see its diff."]
+ for rec in result.added:
+ lines.append("+ added %s %-9s %s" % (rec.guid, rec.kind, _where(rec.name, rec.path)))
+ for rec in result.removed:
+ lines.append("- removed %s %-9s %s" % (rec.guid, rec.kind, _where(rec.name, rec.path)))
+ for ch in result.changed:
+ tags = (["[%s]" % ", ".join(ch.labels)] if ch.labels else []) + ["[%s]" % m for m in ch.moves]
+ lines.append("~ changed %s %-9s %s %s" % (ch.guid, ch.kind, _where(ch.name, ch.path), " ".join(tags)))
+ lines.append("= %d unchanged, %d not compared (kind other or unreadable); %d added, %d removed, %d changed"
+ % (result.unchanged, result.not_compared, len(result.added), len(result.removed), len(result.changed)))
+ return "\n".join(lines)
+
+
+def object_diff_to_dict(result: ObjectDiff) -> dict:
+ return {
+ "guid": result.new.guid, "name": result.new.name, "old_name": result.old.name,
+ "old_kind": result.old.kind, "new_kind": result.new.kind, "notes": result.notes,
+ "sections": [{"label": s.label, "diff": s.lines} for s in result.sections],
+ }
+
+
+def _record_dict(guid: str, name: Optional[str], kind: str, path: List[str]) -> dict:
+ return {"guid": guid, "name": name, "kind": kind, "path": path}
+
+
+def project_diff_to_dict(result: ProjectDiff) -> dict:
+ return {
+ "added": [_record_dict(r.guid, r.name, r.kind, r.path) for r in result.added],
+ "removed": [_record_dict(r.guid, r.name, r.kind, r.path) for r in result.removed],
+ "changed": [dict(_record_dict(c.guid, c.name, c.kind, c.path), labels=c.labels, moves=c.moves)
+ for c in result.changed],
+ "unchanged": result.unchanged, "not_compared": result.not_compared, "notes": result.notes,
+ }
diff --git a/scripts/codesysproj/export.py b/scripts/codesysproj/export.py
new file mode 100644
index 0000000..76f3648
--- /dev/null
+++ b/scripts/codesysproj/export.py
@@ -0,0 +1,53 @@
+"""dump-st: write every ST object's declaration and implementation as text files."""
+
+from __future__ import annotations
+
+import os
+import re
+from collections import defaultdict
+from typing import Dict, List, Tuple
+
+from .project import KIND_DUT, KIND_GVL, KIND_ST, ObjectRecord, Project
+
+MAX_STEM_PART = 80
+_UNSAFE = re.compile(r"[^\w.\- ]", re.UNICODE)
+
+
+def safe_filename_part(text: str) -> str:
+ """Make untrusted text usable as one path component: no separators, no leading dot."""
+ cleaned = _UNSAFE.sub("_", text).strip(". ")[:MAX_STEM_PART]
+ return cleaned or "_"
+
+
+def _stem(rec: ObjectRecord) -> str:
+ if rec.name is None:
+ return "unnamed_" + rec.guid
+ return "%s_%s" % (safe_filename_part(rec.device or "root"), safe_filename_part(rec.name))
+
+
+def dump_st(project: Project, outdir: str) -> List[Tuple[ObjectRecord, List[str]]]:
+ """Write `_.decl.st` / `.impl.st` for every ST POU, and `.decl.st` alone for every GVL
+ and DUT (they have no implementation); return what was written.
+
+ Names that collide (two devices with the same POU name, case-insensitively) all get
+ `_`, so no file is overwritten within a run.
+ """
+ st_objects = [r for r in project.records() if r.kind in (KIND_ST, KIND_GVL, KIND_DUT)]
+ groups: Dict[str, int] = defaultdict(int)
+ for rec in st_objects:
+ groups[_stem(rec).lower()] += 1
+ os.makedirs(outdir, exist_ok=True)
+ written: List[Tuple[ObjectRecord, List[str]]] = []
+ for rec in st_objects:
+ stem = _stem(rec)
+ if groups[stem.lower()] > 1:
+ stem = "%s_%s" % (stem, rec.guid[:8])
+ files = []
+ parts = [(".decl.st", rec.declaration)] + ([(".impl.st", rec.implementation)] if rec.kind == KIND_ST else [])
+ for suffix, text in parts:
+ target = os.path.join(outdir, stem + suffix)
+ with open(target, "w", encoding="utf-8", newline="") as handle:
+ handle.write(text or "")
+ files.append(target)
+ written.append((rec, files))
+ return written
diff --git a/scripts/codesysproj/graph.py b/scripts/codesysproj/graph.py
new file mode 100644
index 0000000..a6c9f54
--- /dev/null
+++ b/scripts/codesysproj/graph.py
@@ -0,0 +1,248 @@
+"""Decode the records of a graphical (LD/FBD) object into a flat, offset-ordered list.
+
+Reverse-engineered from Machine Expert V2.x projects whose graphical POUs carry the editor
+model string `Ld`. Nothing here is documented by the vendor. The split is deliberate:
+
+* DECODED (read at fixed positions inside a recognised record, strings looked up in the
+ string table): network comments, element name and type, box type and instance, the box
+ header id, the modifier byte, pin names, the count byte of a coil record.
+* HEURISTIC (an interpretation, marked `heuristic` in the output): IN or OUT direction
+ (the byte before the element), the meaning of a modifier byte, which OUT elements are
+ coils, box nesting (a box spans from its header id H to the next 8-byte id H+1), and the
+ `ids` lists: qword-shaped runs found by scanning, where a one-byte id (below 256) can
+ come out shifted by a byte (value times 256 plus the byte before it).
+* NOT DECODED: which id connects to which, i.e. the wiring.
+"""
+
+from __future__ import annotations
+
+import struct
+from dataclasses import dataclass, field
+from typing import Dict, List, Optional, Tuple
+
+from .binary import FormatError, read_varint
+from .project import NETWORK_MARKERS, ObjectRecord
+
+SENTINEL_ID = 733 # a constant filler id seen in every record of every project examined
+MAX_ID = 0xFFFF # ids are 8-byte little-endian qwords; only the low 16 bits were ever set
+QWORD = 8
+HEADER_LOOKAHEAD = 60 # bytes after a box's type/instance in which its header id must appear
+MODIFIER_LOOKAHEAD = 40
+BOX_TAGS = (0xE0, 0xE2, 0xE3, 0xE4, 0xE5) # each followed by 02
+ELEMENT_TAG = bytes.fromhex("df02")
+INPUT_PREFIX = 0x69
+COILS_TAG = bytes.fromhex("0f6f6c")
+PARALLEL_TAG = bytes.fromhex("0fa601")
+EMPTY_TAG = bytes.fromhex("9a01")
+EMPTY_PAD = 5 # bytes between the empty-slot tag and its id (not decoded; shown raw)
+INSTANCE_NONE = bytes.fromhex("de0200")
+INSTANCE_TAG = bytes.fromhex("df02")
+HEADER_ID_TAG = bytes.fromhex("dd02")
+PINS_TAG, PINS_MID = 0x6A, 0x0E
+EMPTY_STRING_INDEX = 0x27 # string-table index 39 is "" in every project examined
+COMMENT_SKIP = 3 + 5 # network marker, then five bytes, then the comment varint
+MODIFIER_MEANING = {0: "none", 1: "negated", 2: "set", 3: "reset"} # unverified, see reference
+
+
+@dataclass
+class Item:
+ offset: int
+ kind: str # element | box | coils | parallel_marker | empty | pins | unknown
+ decoded: Dict[str, object] = field(default_factory=dict)
+ heuristic: Dict[str, object] = field(default_factory=dict)
+ ids: List[int] = field(default_factory=list)
+
+
+@dataclass
+class Network:
+ offset: int
+ comment: Optional[str]
+ tag: str # "e1 02" or "e7 02": two network record tags, difference not decoded
+ items: List[Item] = field(default_factory=list)
+ ids: List[int] = field(default_factory=list)
+
+
+@dataclass
+class GraphDecode:
+ guid: str
+ name: Optional[str]
+ language: Optional[str]
+ networks: List[Network] = field(default_factory=list)
+ notes: List[str] = field(default_factory=list)
+
+
+def _string_at(data: bytes, pos: int, strings: Dict[int, str]) -> Tuple[Optional[str], int, int]:
+ """(text, string index, position after). Text is None when the index is not in the table."""
+ index, after = read_varint(data, pos)
+ return strings.get(index), index, after
+
+
+def _is_id(data: bytes, pos: int) -> bool:
+ return (pos + QWORD <= len(data) and data[pos + 2:pos + QWORD] == b"\x00" * 6
+ and (data[pos] != 0 or data[pos + 1] != 0))
+
+
+def _parse_box(data: bytes, pos: int, strings: Dict[int, str]) -> Optional[Tuple[Item, int]]:
+ if pos + 1 >= len(data) or data[pos] not in BOX_TAGS or data[pos + 1] != 2:
+ return None
+ try:
+ box_type, _, after = _string_at(data, pos + 2, strings)
+ if data[after:after + 3] == INSTANCE_NONE:
+ instance: Optional[str] = ""
+ after += 3
+ elif data[after:after + 2] == INSTANCE_TAG:
+ instance, _, after = _string_at(data, after + 2, strings)
+ else:
+ return None
+ except FormatError:
+ return None
+ if box_type is None:
+ return None
+ window = data[after:after + HEADER_LOOKAHEAD]
+ found = window.find(HEADER_ID_TAG)
+ while found != -1:
+ start = found + len(HEADER_ID_TAG) + QWORD
+ if window[start:start + 1] == b"\x01" and len(window) >= start + 1 + QWORD:
+ header_id = struct.unpack_from(" Tuple[Item, int]:
+ return Item(pos, "unknown", {"reason": reason, "hex": data[pos:pos + 16].hex(" ")}), after
+
+
+def _parse_element(data: bytes, pos: int, strings: Dict[int, str]) -> Tuple[Item, int]:
+ direction = "IN" if pos > 0 and data[pos - 1] == INPUT_PREFIX else "OUT"
+ item = Item(pos, "element", {}, {"direction": direction})
+ after = pos + len(ELEMENT_TAG)
+ try:
+ if data[after] == EMPTY_STRING_INDEX:
+ item.decoded["name"] = ""
+ after += 1
+ else:
+ name, index, after = _string_at(data, after, strings)
+ if name is None:
+ return _unknown(data, pos, "element name index %d is not in the string table" % index,
+ pos + len(ELEMENT_TAG))
+ item.decoded["name"] = name
+ if after < len(data) and data[after] != EMPTY_STRING_INDEX:
+ item.decoded["type"], _, after = _string_at(data, after, strings)
+ except (FormatError, IndexError):
+ return _unknown(data, pos, "element record runs past the end of the object", len(data))
+ at = data.find(HEADER_ID_TAG, after, after + MODIFIER_LOOKAHEAD)
+ if at != -1 and at + 2 < len(data):
+ item.decoded["modifier"] = data[at + 2]
+ item.heuristic["modifier_meaning"] = MODIFIER_MEANING.get(data[at + 2], "unknown code")
+ else:
+ item.decoded["modifier"] = None
+ return item, after
+
+
+def _parse_pins(data: bytes, pos: int, strings: Dict[int, str]) -> Optional[Tuple[Item, int]]:
+ if pos + 3 > len(data) or data[pos] != PINS_TAG or data[pos + 2] != PINS_MID:
+ return None
+ names: List[Optional[str]] = []
+ after = pos + 3
+ try:
+ for _ in range(data[pos + 1]):
+ text, _, after = _string_at(data, after, strings)
+ names.append(text)
+ except FormatError:
+ return None
+ return Item(pos, "pins", {"names": names}), after
+
+
+def _next_item(data: bytes, pos: int, strings: Dict[int, str]) -> Optional[Tuple[Item, int]]:
+ if data.startswith(COILS_TAG, pos) and pos + 4 <= len(data):
+ return Item(pos, "coils", {"count": data[pos + 3]}), pos + 4
+ if data.startswith(PARALLEL_TAG, pos) and pos + 4 <= len(data):
+ return Item(pos, "parallel_marker", {"value": data[pos + 3]}), pos + 4
+ if data.startswith(EMPTY_TAG, pos) and _is_id(data, pos + 2 + EMPTY_PAD):
+ id_at = pos + 2 + EMPTY_PAD
+ return Item(pos, "empty", {"raw": data[pos + 2:id_at].hex(" "),
+ "id": struct.unpack_from(" None:
+ pos = start
+ owner: List[int] = net.ids
+ coils_left = 0
+ while pos < len(data):
+ found = _next_item(data, pos, strings)
+ if found is not None:
+ item, after = found
+ if item.kind == "coils":
+ coils_left = int(item.decoded["count"])
+ elif item.kind == "element" and coils_left and item.heuristic["direction"] == "OUT":
+ item.heuristic["role"] = "coil"
+ coils_left -= 1
+ net.items.append(item)
+ owner = item.ids
+ pos = max(after, pos + 1)
+ elif _is_id(data, pos):
+ value = struct.unpack_from(" None:
+ spans = [(i.offset, i.heuristic["span_end"]) for i in net.items
+ if i.kind == "box" and "span_end" in i.heuristic]
+ for item in net.items:
+ item.heuristic["depth"] = sum(1 for lo, hi in spans if lo < item.offset < hi)
+
+
+def _network_starts(data: bytes) -> List[int]:
+ starts: List[int] = []
+ for marker in NETWORK_MARKERS:
+ at = data.find(marker)
+ while at != -1:
+ starts.append(at)
+ at = data.find(marker, at + 1)
+ return sorted(starts)
+
+
+def decode_graph(rec: ObjectRecord, data: bytes, strings: Dict[int, str]) -> GraphDecode:
+ """Decode networks and their records. Unrecognised bytes are skipped silently except
+ where a record was recognised but could not be read: those become `unknown` items."""
+ result = GraphDecode(rec.guid, rec.name, rec.language)
+ starts = _network_starts(data)
+ if not starts:
+ result.notes.append("no network records found; not a graphical object in the observed layout")
+ return result
+ for number, start in enumerate(starts):
+ end = starts[number + 1] if number + 1 < len(starts) else len(data)
+ segment = data[:end]
+ comment: Optional[str] = None
+ body = start + COMMENT_SKIP
+ if body < len(segment) and segment[body] != 0:
+ try:
+ comment, index, body = _string_at(segment, body, strings)
+ if comment is None:
+ result.notes.append("network %d: comment index %d is not in the string table" % (number + 1, index))
+ except FormatError:
+ result.notes.append("network %d: record ends inside the comment" % (number + 1))
+ body = end
+ net = Network(start, comment, data[start + 1:start + 3].hex(" "))
+ _scan_network(segment, body, net, strings)
+ _box_depths(net)
+ result.networks.append(net)
+ unknown = sum(1 for n in result.networks for i in n.items if i.kind == "unknown")
+ if unknown:
+ result.notes.append("%d record(s) could not be decoded; see the `unknown` items" % unknown)
+ return result
diff --git a/scripts/codesysproj/graphview.py b/scripts/codesysproj/graphview.py
new file mode 100644
index 0000000..6cd2e8e
--- /dev/null
+++ b/scripts/codesysproj/graphview.py
@@ -0,0 +1,68 @@
+"""Text and JSON renderings of a decoded graph (see graph.py for what is decoded and what is a heuristic)."""
+
+from __future__ import annotations
+
+from .graph import GraphDecode, Item
+
+
+def graph_to_dict(result: GraphDecode) -> dict:
+ return {
+ "guid": result.guid, "name": result.name, "language": result.language,
+ "legend": "decoded = read from the record by the observed layout; heuristic = interpretation, "
+ "including every ids list; wiring (which id connects to which) is not decoded; bytes that match no "
+ "recognised record are skipped without a note, so a clean run is not full coverage",
+ "notes": result.notes,
+ "networks": [
+ {"offset": n.offset, "tag": n.tag, "comment": n.comment, "heuristic": {"ids": n.ids},
+ "items": [{"offset": i.offset, "kind": i.kind, "decoded": i.decoded,
+ "heuristic": dict(i.heuristic, ids=i.ids)} for i in n.items]}
+ for n in result.networks
+ ],
+ }
+
+
+def item_label(item: Item, stable: bool = False) -> str:
+ """One item as text, indented by box depth. `stable` leaves out everything that changes when the
+ project is saved again without a logic change: header ids, empty-slot bytes and ids, raw bytes."""
+ d, h = item.decoded, item.heuristic
+ indent = " " * int(h.get("depth", 0))
+ if item.kind == "element":
+ text = "%s %r" % (h["direction"] + ("/coil" if h.get("role") else ""), d["name"])
+ if d.get("type"):
+ text += " type=%s" % d["type"]
+ text += " mod=%s" % d["modifier"]
+ if h.get("modifier_meaning") not in (None, "none"):
+ text += " ~%s" % h["modifier_meaning"]
+ elif item.kind == "box":
+ text = "BOX %s" % d["box_type"] + (" instance=%s" % d["instance"] if d["instance"] else "")
+ text += "" if stable else " id=%s" % d["header_id"]
+ elif item.kind == "unknown":
+ text = "UNKNOWN %s" % d["reason"] + ("" if stable else " [%s]" % d["hex"])
+ elif item.kind == "empty" and stable:
+ text = "empty"
+ else:
+ text = "%s %s" % (item.kind, " ".join("%s=%s" % kv for kv in d.items()))
+ return indent + text
+
+
+def _item_text(item: Item) -> str:
+ ids = " ids~%s" % item.ids if item.ids else ""
+ return "%5d %s%s" % (item.offset, item_label(item), ids)
+
+
+def render_graph_text(result: GraphDecode) -> str:
+ lines = [
+ "# graphical object %s name=%s language=%s (heuristic)" % (result.guid, result.name, result.language),
+ "# decoded: names, types, modifier byte, box ids, comments. '~' marks a heuristic: direction "
+ "(IN/OUT), '/coil', modifier meaning, indentation = box nesting, ids~ (a small id can be shifted by a byte).",
+ "# NOT decoded: wiring (which id connects to which). Verify wiring in the IDE or a PLCopen XML export.",
+ "# Bytes that match no recognised record are skipped without a note: a clean run is not full coverage.",
+ ]
+ lines += ["# note: " + note for note in result.notes]
+ for number, net in enumerate(result.networks, 1):
+ comment = "none" if net.comment is None else repr(net.comment)
+ lines.append("")
+ lines.append("network %d @%d tag=%s comment=%s%s" % (number, net.offset, net.tag, comment,
+ " ids~%s" % net.ids if net.ids else ""))
+ lines += [_item_text(i) for i in net.items]
+ return "\n".join(lines)
diff --git a/scripts/codesysproj/project.py b/scripts/codesysproj/project.py
new file mode 100644
index 0000000..cad9990
--- /dev/null
+++ b/scripts/codesysproj/project.py
@@ -0,0 +1,285 @@
+"""Open a `.project` zip: object inventory, parent chains, kinds and ST text.
+
+Observed container layout (reverse-engineered; see the reference page):
+
+* the `.project` is an ordinary zip; entries are flat: `.meta`, `.object`,
+ `_decl_lineids.auxiliary`, ..., and one shared string table;
+* `.meta` is a varint stream; the string-table text of one value equals the object's own
+ guid, and the next two values are the parent guid and the object name;
+* `.object` starts with a 20-byte header (magic 02 20 09 28, then a uint32 at offset 16
+ holding the number of bytes that follow). Source text is not in the object: the object
+ holds string-table indices. A text slot is ` e5 02 `
+ where the reference string is `_Decl_LineIds` (declaration, marker 41) or
+ `_Impl_LineIds` (implementation, marker 5b).
+
+What is a fact and what is a guess is spelled out per field on `ObjectRecord`.
+"""
+
+from __future__ import annotations
+
+import re
+import struct
+import zipfile
+from dataclasses import dataclass, field
+from typing import Dict, List, Optional, Tuple
+
+from .binary import FormatError, parse_string_table, read_varint
+from . import zipread
+from .zipread import entry_name, open_archive, read_entry
+
+STRING_TABLE_NAME = "__shared_data_storage_string_table__.auxiliary"
+OBJECT_MAGIC = bytes.fromhex("02200928")
+OBJECT_HEADER_LEN = 20
+LENGTH_FIELD_OFFSET = 16
+DECL_MARKER = bytes.fromhex("41e502")
+IMPL_MARKER = bytes.fromhex("5be502")
+DECL_REF_SUFFIX = "_Decl_LineIds"
+IMPL_REF_SUFFIX = "_Impl_LineIds"
+NETWORK_MARKERS = (bytes.fromhex("0fe102"), bytes.fromhex("0fe702")) # two network record tags
+LANGUAGE_PREFIX = bytes.fromhex("0fe60248")
+LANGUAGE_VARINT_INDEX = 5 # counted in varints from the start of the payload
+PLC_LOGIC_NAME = "plc logic"
+
+GUID_RE = re.compile(r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$")
+_DECL_START_RE = re.compile(
+ r"\s*(?:(?://[^\n]*\n|\(\*(?:(?!\*\)).)*\*\)|\{[^}]*\})\s*)*"
+ r"(PROGRAM|FUNCTION_BLOCK|FUNCTION|METHOD|PROPERTY|ACTION|INTERFACE|TRANSITION|TYPE"
+ r"|VAR_GLOBAL|VAR_CONFIG|VAR)\b",
+ re.S,
+)
+
+KIND_ST = "st"
+KIND_GRAPHICAL = "graphical"
+KIND_GVL = "gvl"
+KIND_DUT = "dut"
+KIND_OTHER = "other"
+KIND_UNREADABLE = "unreadable"
+KINDS = (KIND_ST, KIND_GRAPHICAL, KIND_GVL, KIND_DUT, KIND_OTHER, KIND_UNREADABLE)
+
+
+@dataclass
+class ObjectRecord:
+ """One object of the project.
+
+ Decoded (read from the bytes by the observed layout): guid, name, parent_guid, path,
+ declaration, implementation, problems. Heuristic (a classification of those bytes):
+ kind, subtype, language, device, application.
+ """
+
+ guid: str
+ name: Optional[str] = None
+ parent_guid: Optional[str] = None
+ path: List[str] = field(default_factory=list) # ancestor names, root first
+ kind: str = KIND_OTHER
+ subtype: Optional[str] = None # the declaration keyword: PROGRAM, FUNCTION_BLOCK, ...
+ language: Optional[str] = None # graphical objects: the editor-model string
+ declaration: Optional[str] = None
+ implementation: Optional[str] = None
+ has_meta: bool = False
+ has_object: bool = False
+ problems: List[str] = field(default_factory=list)
+
+ @property
+ def device(self) -> Optional[str]:
+ """Root-most ancestor (heuristic: the controller / device node)."""
+ return self.path[0] if self.path else None
+
+ @property
+ def application(self) -> Optional[str]:
+ """The node under 'Plc Logic' (heuristic; None when no ancestor has that name)."""
+ for i, name in enumerate(self.path[:-1]):
+ if name.lower() == PLC_LOGIC_NAME:
+ return self.path[i + 1]
+ return None
+
+
+@dataclass
+class Project:
+ path: str
+ strings: Dict[int, str]
+ objects: Dict[str, ObjectRecord]
+ blobs: Dict[str, bytes] # guid -> raw .object bytes
+
+ def records(self) -> List[ObjectRecord]:
+ """Objects in tree order (by ancestor names, then name, then guid)."""
+ return sorted(self.objects.values(), key=lambda r: (r.path, r.name or "", r.guid))
+
+
+def _scan_varints(data: bytes, start: int = 0) -> List[int]:
+ """Varints from `start` until the data stops being a clean varint stream."""
+ values: List[int] = []
+ pos = start
+ while pos < len(data):
+ try:
+ value, pos = read_varint(data, pos)
+ except FormatError:
+ break
+ values.append(value)
+ return values
+
+
+def parse_meta(data: bytes, guid: str, strings: Dict[int, str]) -> Tuple[str, str]:
+ """Return (parent guid, name) read after the object's own guid in the varint stream."""
+ texts = [strings.get(v) for v in _scan_varints(data)]
+ if guid not in texts:
+ raise FormatError("own guid not found in the meta stream")
+ at = texts.index(guid)
+ while at + 1 < len(texts) and texts[at + 1] == guid: # some objects repeat their own guid
+ at += 1
+ parent, name = (texts[at + 1], texts[at + 2]) if at + 2 < len(texts) else (None, None)
+ if parent is None or not GUID_RE.match(parent) or name is None:
+ raise FormatError("own guid is not followed by a parent guid and a name")
+ return parent, name
+
+
+def _text_slot(data: bytes, marker: bytes, suffix: str, guid: str, strings: Dict[int, str],
+ label: str, problems: List[str]) -> Optional[str]:
+ """Text behind the slot `marker e5 02 `; None if absent or undecodable.
+
+ A marker whose reference string is not `` is data that happens to look
+ like the marker, and is ignored.
+ """
+ hits: List[int] = []
+ at = data.find(marker, OBJECT_HEADER_LEN)
+ while at != -1:
+ try:
+ text_index, pos = read_varint(data, at + len(marker))
+ ref_index, _ = read_varint(data, pos)
+ except FormatError:
+ text_index = ref_index = -1
+ if strings.get(ref_index) == guid + suffix:
+ hits.append(text_index)
+ at = data.find(marker, at + 1)
+ if not hits:
+ return None
+ if len(set(hits)) > 1:
+ problems.append("%d %s slots with different text; using the first" % (len(set(hits)), label))
+ if hits[0] not in strings:
+ problems.append("%s text index %d is not in the string table" % (label, hits[0]))
+ return None
+ return strings[hits[0]]
+
+
+def _language(data: bytes, strings: Dict[int, str]) -> Optional[str]:
+ payload = data[OBJECT_HEADER_LEN:]
+ if not payload.startswith(LANGUAGE_PREFIX):
+ return None
+ values = _scan_varints(payload)
+ if len(values) <= LANGUAGE_VARINT_INDEX:
+ return None
+ return strings.get(values[LANGUAGE_VARINT_INDEX])
+
+
+def declaration_keyword(text: str) -> Optional[str]:
+ match = _DECL_START_RE.match(text)
+ return match.group(1) if match else None
+
+
+def classify_object(rec: ObjectRecord, data: bytes, strings: Dict[int, str]) -> None:
+ """Fill kind, subtype, language, declaration and implementation from `.object` bytes."""
+ if len(data) < OBJECT_HEADER_LEN:
+ rec.kind = KIND_UNREADABLE
+ rec.problems.append("object is %d bytes, shorter than the %d-byte header" % (len(data), OBJECT_HEADER_LEN))
+ return
+ if data[:4] != OBJECT_MAGIC:
+ rec.kind = KIND_UNREADABLE
+ rec.problems.append("object does not start with the magic 02 20 09 28 (starts %s)" % data[:4].hex(" "))
+ return
+ declared = struct.unpack_from(" None:
+ for rec in objects.values():
+ chain: List[str] = []
+ seen = {rec.guid}
+ parent = rec.parent_guid
+ while parent in objects and parent not in seen:
+ seen.add(parent)
+ chain.append(objects[parent].name or parent)
+ parent = objects[parent].parent_guid
+ if parent in seen:
+ rec.problems.append("parent chain loops back on itself")
+ rec.path = list(reversed(chain))
+
+
+def load_project(path: str) -> Project:
+ """Read every object of the `.project`. Per-object problems go to `ObjectRecord.problems`;
+ a file that is not a project at all raises FormatError."""
+ archive = open_archive(path)
+ with archive:
+ entries: Dict[str, Dict[str, zipfile.ZipInfo]] = {}
+ table_info = None
+ for info in archive.infolist():
+ name = entry_name(info)
+ if name == STRING_TABLE_NAME:
+ table_info = info
+ continue
+ stem, dot, ext = name.rpartition(".")
+ if dot and ext in ("meta", "object") and GUID_RE.match(stem):
+ entries.setdefault(stem, {})[ext] = info
+ if table_info is None:
+ raise FormatError("%s: no %s entry; not a CODESYS 3.5 project" % (path, STRING_TABLE_NAME))
+ strings = parse_string_table(read_entry(archive, table_info))
+ objects: Dict[str, ObjectRecord] = {}
+ blobs: Dict[str, bytes] = {}
+ total = 0
+ for guid, parts in entries.items():
+ rec = ObjectRecord(guid=guid, has_meta="meta" in parts, has_object="object" in parts)
+ objects[guid] = rec
+ if rec.has_meta:
+ try:
+ rec.parent_guid, rec.name = parse_meta(read_entry(archive, parts["meta"]), guid, strings)
+ except FormatError as exc:
+ rec.problems.append("meta: %s" % exc)
+ else:
+ rec.problems.append("no .meta entry: name and parent unknown")
+ if rec.has_object:
+ try:
+ blobs[guid] = read_entry(archive, parts["object"])
+ total += len(blobs[guid])
+ except FormatError as exc:
+ rec.kind = KIND_UNREADABLE
+ rec.problems.append("object: %s" % exc)
+ else:
+ if total > zipread.TOTAL_CAP:
+ raise FormatError("%s: objects inflate past %d bytes in total; not read" % (path, zipread.TOTAL_CAP))
+ classify_object(rec, blobs[guid], strings)
+ else:
+ rec.problems.append("no .object entry")
+ _link_parents(objects)
+ return Project(path=path, strings=strings, objects=objects, blobs=blobs)
+
+
+def find_objects(project: Project, query: str) -> List[ObjectRecord]:
+ """Objects matching a full guid, a unique guid prefix (8+ chars), or a name (IEC names ignore case)."""
+ exact = project.objects.get(query.lower())
+ if exact is not None:
+ return [exact]
+ if len(query) >= 8:
+ prefix = [r for g, r in project.objects.items() if g.startswith(query.lower())]
+ if prefix:
+ return sorted(prefix, key=lambda r: r.guid)
+ return [r for r in project.records() if (r.name or "").lower() == query.lower()]
diff --git a/scripts/codesysproj/stscan.py b/scripts/codesysproj/stscan.py
new file mode 100644
index 0000000..7dc1056
--- /dev/null
+++ b/scripts/codesysproj/stscan.py
@@ -0,0 +1,135 @@
+"""Scanning ST source text for identifier paths: noise removal and a path scanner.
+
+Used by xref.py. The scanner reads text, not a grammar: it finds `a.b[c].d` style paths, skips numbers,
+typed literals and addresses, and reports each path with its position. What a path means is decided by
+the caller.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from typing import List
+
+IDENT_RE = re.compile(r"[^\W\d]\w*") # a letter or `_`, then letters, digits, `_` (any script)
+_LITERAL_TAIL_RE = re.compile(r"[A-Za-z0-9_#.]*")
+_ADDRESS_RE = re.compile(r"%[A-Za-z0-9_.]*")
+
+
+@dataclass
+class NameRef:
+ segments: List[str] # as written, indexes removed
+ start: int
+ end: int
+ in_index: bool
+
+
+def strip_noise(text: str) -> str:
+ """Comments (`(* *)` nest, `//`), pragmas `{ }` and string literals become spaces; newlines and
+ every position stay, so an offset in the result is an offset in the text. Unterminated ones run to the end."""
+ out = list(text)
+ n, i, depth = len(text), 0, 0
+ while i < n:
+ two = text[i:i + 2]
+ if two == "(*":
+ depth += 1
+ start, i = i, i + 2
+ while i < n and depth:
+ if text.startswith("(*", i):
+ depth, i = depth + 1, i + 2
+ elif text.startswith("*)", i):
+ depth, i = depth - 1, i + 2
+ else:
+ i += 1
+ _blank(out, start, i)
+ elif two == "//":
+ start = i
+ i = text.find("\n", i)
+ i = n if i == -1 else i
+ _blank(out, start, i)
+ elif text[i] == "{":
+ start = i
+ i = text.find("}", i)
+ i = n if i == -1 else i + 1
+ _blank(out, start, i)
+ elif text[i] in "'\"":
+ start, quote = i, text[i]
+ i += 1
+ while i < n and text[i] != quote:
+ i += 2 if text[i] == "$" else 1
+ i = min(i + 1, n)
+ _blank(out, start, i)
+ else:
+ i += 1
+ return "".join(out)
+
+
+def _blank(out: List[str], start: int, end: int) -> None:
+ for k in range(start, min(end, len(out))):
+ if out[k] != "\n":
+ out[k] = " "
+
+
+def _matching_bracket(code: str, at: int) -> int:
+ depth = 0
+ for k in range(at, len(code)):
+ depth += code[k] == "["
+ depth -= code[k] == "]"
+ if depth == 0:
+ return k
+ return len(code) - 1
+
+
+def scan_paths(code: str, base: int = 0, in_index: bool = False) -> List[NameRef]:
+ """Every identifier path in (noise-free) code: `a.b[c].d` is one path `a.b.d` plus the path `c`
+ found inside the index. Numbers, typed literals (`T#5s`, `16#FF`) and addresses are skipped."""
+ paths: List[NameRef] = []
+ n, i = len(code), 0
+ while i < n:
+ c = code[i]
+ if c.isdigit():
+ i = _skip_number(code, i)
+ elif c == "%":
+ i = _ADDRESS_RE.match(code, i).end()
+ elif IDENT_RE.match(c):
+ i = _scan_one_path(code, i, base, in_index, paths)
+ else:
+ i += 1
+ return paths
+
+
+def _skip_number(code: str, i: int) -> int:
+ while i < len(code):
+ if code[i].isalnum() or code[i] in "_#":
+ i += 1
+ elif code[i] == "." and code[i + 1:i + 2].isdigit():
+ i += 1
+ else:
+ break
+ return i
+
+
+def _scan_one_path(code: str, i: int, base: int, in_index: bool, paths: List[NameRef]) -> int:
+ start, segments = i, []
+ nested: List[NameRef] = []
+ while True:
+ match = IDENT_RE.match(code, i)
+ segments.append(match.group())
+ i = match.end()
+ while i < len(code) and code[i] in "[^":
+ if code[i] == "[":
+ close = _matching_bracket(code, i)
+ nested += scan_paths(code[i + 1:close], base + i + 1, True)
+ i = close + 1
+ else:
+ i += 1
+ if code[i:i + 1] == "#": # a typed literal such as REAL#1.0, not a variable
+ return _LITERAL_TAIL_RE.match(code, i).end()
+ follows = code[i + 1:i + 2]
+ if code[i:i + 1] == "." and IDENT_RE.match(follows):
+ i += 1
+ continue
+ break
+ paths.append(NameRef(segments, base + start, base + i, in_index))
+ paths.extend(nested)
+ return i
diff --git a/scripts/codesysproj/xref.py b/scripts/codesysproj/xref.py
new file mode 100644
index 0000000..9de14db
--- /dev/null
+++ b/scripts/codesysproj/xref.py
@@ -0,0 +1,271 @@
+"""Where a variable, instance or type name is used: ST text and decoded LD/FBD elements.
+
+EVERY result is a heuristic. The match is by name only (IEC 61131-3 is case-insensitive), with no
+scope resolution: a local variable and a global with the same name both match, and `x.Foo` is
+reported for the query `Foo` as well as `Foo.x`. A query of several segments (`Gvl.Foo`) matches
+paths that contain those segments in a row.
+
+Roles, per reference:
+
+* ST implementation: `write` is the target before `:=` outside a call, or the variable after `=>`;
+ `read` is any other use (an array index of a target is a read); `call` is a name followed by `(`;
+ `param` is the formal name before `:=` or `=>` inside a call; `arg` is the value after `name :=`
+ inside a call, whose direction depends on the callee's declaration, which is not read.
+* Declarations: `declared` (the name before the `:`), `type` (the part after it, up to `:=`),
+ `read` (an initial value).
+* LD/FBD: `write` for an element the decoder labels OUT, `read` for IN (both heuristic: the byte
+ before the element), `read` for names inside an index, `type` for a box type, `call` for a box
+ instance. Wiring is not decoded.
+
+Not searched: objects of kind other (devices, tasks, I/O mappings: a variable mapped to a channel
+there is not found), and ST text that the object's slots do not carry. Graphical records the
+decoder could not read are counted, since a reference may hide in one.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+from typing import List, Optional, Tuple
+
+from .binary import FormatError
+from .graph import decode_graph
+from .project import KIND_DUT, KIND_GRAPHICAL, KIND_GVL, KIND_ST, ObjectRecord, Project
+from .stscan import IDENT_RE, NameRef, scan_paths, strip_noise
+
+_SECTION_WORDS = frozenset(w.lower() for w in (
+ "VAR", "VAR_INPUT", "VAR_OUTPUT", "VAR_IN_OUT", "VAR_GLOBAL", "VAR_TEMP", "VAR_STAT", "VAR_EXTERNAL",
+ "VAR_CONFIG", "VAR_INST", "CONSTANT", "RETAIN", "PERSISTENT", "END_VAR", "STRUCT", "END_STRUCT"))
+_HEADER_RE = re.compile(
+ r"\s*(?:FUNCTION_BLOCK|FUNCTION|METHOD|PROPERTY|PROGRAM|INTERFACE|TYPE|ACTION|TRANSITION)\b[ \t]*([^\W\d]\w*)?",
+ re.I)
+_STRUCT_RE = re.compile(r"\bSTRUCT\b", re.I)
+# `FUNCTION_BLOCK F EXTENDS B IMPLEMENTS I`, each clause on the header line or on a line of its own
+_INHERIT_RE = re.compile(r"(?:\s+(?:EXTENDS|IMPLEMENTS)\b[^\n:]*)+", re.I)
+_INHERIT_WORD_RE = re.compile(r"\b(?:EXTENDS|IMPLEMENTS)\b", re.I)
+SEARCHED_KINDS = (KIND_ST, KIND_GRAPHICAL, KIND_GVL, KIND_DUT)
+
+
+@dataclass
+class Reference:
+ guid: str
+ name: Optional[str]
+ kind: str
+ path: List[str]
+ where: str # "implementation:12", "declaration:3", "network 2"
+ role: str
+ text: str # the source line, or the element label
+
+
+@dataclass
+class XrefResult:
+ query: str
+ hits: List[Reference] = field(default_factory=list)
+ searched: int = 0
+ not_searched: int = 0
+ notes: List[str] = field(default_factory=list)
+
+
+def _contains(segments: List[str], query: List[str]) -> bool:
+ low = [s.lower() for s in segments]
+ return any(low[k:k + len(query)] == query for k in range(len(low) - len(query) + 1))
+
+
+def _st_role(code: str, path: NameRef) -> str:
+ """Role of an ST path from the text around it (`code` is noise-free)."""
+ head = code[:path.start]
+ before = head.rstrip()
+ after = code[path.end:].lstrip()
+ in_call = head.count("(") > head.count(")")
+ if after.startswith("=>"):
+ return "param"
+ if after.startswith(":="):
+ return "param" if in_call else "write"
+ if after.startswith("("):
+ return "call"
+ if before.endswith("=>"):
+ return "write"
+ if before.endswith(":=") and in_call:
+ return "arg"
+ return "read"
+
+
+def _line_of(text: str, offset: int) -> int:
+ return text.count("\n", 0, offset) + 1
+
+
+def _source_line(text: str, line: int) -> str:
+ lines = text.splitlines()
+ return lines[line - 1].strip() if 0 < line <= len(lines) else ""
+
+
+def _scan_implementation(rec: ObjectRecord, query: List[str]) -> List[Reference]:
+ text = rec.implementation or ""
+ code = strip_noise(text)
+ refs = []
+ for path in scan_paths(code):
+ if _contains(path.segments, query):
+ line = _line_of(code, path.start)
+ refs.append(Reference(rec.guid, rec.name, rec.kind, rec.path, "implementation:%d" % line,
+ _st_role(code, path), _source_line(text, line)))
+ return refs
+
+
+def _declared_names(left: str) -> List[Tuple[str, int]]:
+ """Variable names, with their offsets, in the text before a declaration's `:`: `VAR x, y AT %MX0.0`
+ gives x and y; a name list may run over several lines."""
+ names, pos = [], 0
+ for part in left.split(","):
+ for token in re.finditer(r"\S+", part):
+ if token.group().lower() not in _SECTION_WORDS:
+ names.append((token.group(), pos + token.start()))
+ break
+ pos += len(part) + 1
+ return names
+
+
+class _Declaration:
+ """Scans one object's declaration text for a query, chunk by chunk (a chunk ends at `;`)."""
+
+ def __init__(self, rec: ObjectRecord, query: List[str]):
+ self.rec, self.query = rec, query
+ self.text = rec.declaration or ""
+ self.code = strip_noise(self.text)
+ self.own_name_ok = len(query) == 1 or (rec.name or "").lower() == query[-2]
+ self.refs: List[Reference] = []
+
+ def add(self, at: int, role: str) -> None:
+ line = _line_of(self.code, at)
+ self.refs.append(Reference(self.rec.guid, self.rec.name, self.rec.kind, self.rec.path,
+ "declaration:%d" % line, role, _source_line(self.text, line)))
+
+ def declared(self, name: str, at: int) -> None:
+ if name.lower() == self.query[-1] and self.own_name_ok:
+ self.add(at, "declared")
+
+ def paths(self, text: str, base: int, role_of) -> None:
+ for path in scan_paths(text, base):
+ if _contains(path.segments, self.query):
+ self.add(path.start, role_of(path))
+
+ def header(self, chunk: str, offset: int) -> int:
+ """Handle a leading `FUNCTION F : T` / `TYPE T : STRUCT` header; return where the ordinary
+ declarations in the chunk start. A header has a type only when its name is followed by the `:`
+ (`PROGRAM P` has none; in `PROGRAM P VAR x : INT` the `:` is a variable's)."""
+ match = _HEADER_RE.match(chunk) if offset == 0 else None # only the first chunk can be the header
+ if match is None:
+ return 0
+ if match.group(1):
+ self.declared(match.group(1), offset + match.start(1))
+ head_end = match.end()
+ inherit = _INHERIT_RE.match(chunk, head_end)
+ if inherit is not None: # the base and the interfaces are types
+ clause = _INHERIT_WORD_RE.sub(lambda m: " " * len(m.group()), chunk[head_end:inherit.end()])
+ self.paths(clause, offset + head_end, lambda path: "type")
+ head_end = inherit.end()
+ first_line_end = chunk.find("\n", len(chunk) - len(chunk.lstrip())) # a pragma line may precede the header
+ colon = chunk.find(":")
+ if colon == -1 or chunk[head_end:colon].strip():
+ return head_end
+ struct = _STRUCT_RE.search(chunk, colon)
+ ends = [e for e in (first_line_end, struct.start() if struct else -1) if e != -1]
+ end = min(ends) if ends else len(chunk)
+ self.paths(chunk[colon + 1:end], offset + colon + 1, lambda path: "type")
+ return end
+
+ def variables(self, chunk: str, offset: int) -> None:
+ colon = chunk.find(":")
+ if colon == -1:
+ return
+ for name, at in _declared_names(chunk[:colon]):
+ self.declared(name, offset + at)
+ right_at = offset + colon + 1
+ init = chunk.find(":=", colon)
+ self.paths(chunk[colon + 1:], right_at,
+ lambda path: "type" if init == -1 or path.start < offset + init else "read")
+
+ def scan(self) -> List[Reference]:
+ offset = 0
+ for chunk in self.code.split(";"):
+ start = self.header(chunk, offset)
+ self.variables(chunk[start:], offset + start)
+ offset += len(chunk) + 1
+ return self.refs
+
+
+def _scan_declaration(rec: ObjectRecord, query: List[str]) -> List[Reference]:
+ return _Declaration(rec, query).scan()
+
+
+def _scan_graph(project: Project, rec: ObjectRecord, query: List[str], result: XrefResult) -> List[Reference]:
+ decoded = decode_graph(rec, project.blobs[rec.guid], project.strings)
+ unreadable = sum(1 for n in decoded.networks for i in n.items if i.kind == "unknown")
+ if unreadable:
+ result.notes.append("%s: %d record(s) could not be decoded; a reference there would be missed"
+ % (rec.name, unreadable))
+ refs: List[Reference] = []
+ for number, net in enumerate(decoded.networks, 1):
+ for item in net.items:
+ for role, label, expression in _graph_names(item):
+ for path in scan_paths(strip_noise(expression)): # a quoted constant is not a name
+ if _contains(path.segments, query):
+ refs.append(Reference(rec.guid, rec.name, rec.kind, rec.path, "network %d" % number,
+ "read" if path.in_index else role,
+ label))
+ return refs
+
+
+def _graph_names(item) -> List[Tuple[str, str, str]]:
+ """(role, label, expression) for each name an item carries."""
+ d = item.decoded
+ if item.kind == "element":
+ label = "%s %r" % (item.heuristic["direction"], d["name"])
+ return [("write" if item.heuristic["direction"] == "OUT" else "read", label, d["name"])]
+ if item.kind == "box":
+ label = "BOX %s" % d["box_type"] + (" instance=%s" % d["instance"] if d["instance"] else "")
+ return [("type", label, d["box_type"]), ("call", label, d["instance"])]
+ return []
+
+
+def find_references(project: Project, query: str) -> XrefResult:
+ """Every reference to `query` (a name or a dotted path) in the searched objects."""
+ segments = [s.strip().lower() for s in query.split(".")]
+ if not all(IDENT_RE.fullmatch(s) for s in segments):
+ raise FormatError("not a name or dotted path: %r" % query)
+ result = XrefResult(query)
+ for rec in project.records():
+ if rec.kind not in SEARCHED_KINDS:
+ result.not_searched += 1
+ continue
+ result.searched += 1
+ result.hits += _scan_declaration(rec, segments)
+ if rec.kind == KIND_ST:
+ result.hits += _scan_implementation(rec, segments)
+ elif rec.kind == KIND_GRAPHICAL:
+ result.hits += _scan_graph(project, rec, segments, result)
+ return result
+
+
+def render_xref_text(result: XrefResult) -> str:
+ lines = [
+ "# references to %r. Heuristic: matched by name (case-insensitive), no scope resolution; roles are read from the text "
+ "around the name (ST) or from the element's IN/OUT label (LD/FBD, itself a heuristic); wiring is not decoded." % result.query,
+ "# not searched: %d object(s) of kind other or unreadable (devices, tasks, I/O mappings: a variable "
+ "mapped to a channel there is not found)." % result.not_searched,
+ ]
+ lines += ["# note: " + n for n in result.notes]
+ for hit in result.hits:
+ lines.append("%s (%s) %s %s %s" % (
+ "/".join(hit.path + [hit.name or "?"]), hit.kind, hit.where, hit.role, hit.text))
+ objects = len({h.guid for h in result.hits})
+ lines.append("%d reference(s) in %d object(s) of %d searched" % (len(result.hits), objects, result.searched))
+ return "\n".join(lines)
+
+
+def xref_to_dict(result: XrefResult) -> dict:
+ return {
+ "query": result.query, "heuristic": "matched by name, no scope resolution; roles are heuristics",
+ "searched": result.searched, "not_searched": result.not_searched, "notes": result.notes,
+ "hits": [{"guid": h.guid, "name": h.name, "kind": h.kind, "path": h.path, "where": h.where,
+ "role": h.role, "text": h.text} for h in result.hits],
+ }
diff --git a/scripts/codesysproj/zipread.py b/scripts/codesysproj/zipread.py
new file mode 100644
index 0000000..d3faa92
--- /dev/null
+++ b/scripts/codesysproj/zipread.py
@@ -0,0 +1,53 @@
+"""Open the `.project` zip and read one entry, with every failure as a FormatError.
+
+The reader is handed files it did not write, so a damaged, encrypted or oversized entry must
+become a message, not a traceback: a CRC error, a corrupt deflate stream, an encrypted entry and
+an entry that inflates past ENTRY_CAP all raise FormatError here. The caller decides whether that
+is fatal (the string table) or one object's problem.
+"""
+
+from __future__ import annotations
+
+import zipfile
+import zlib
+from typing import Optional
+
+from .binary import FormatError
+
+# The largest entry the reader touches in the projects it was checked on is under 3 MB; the cap
+# leaves wide headroom and stops a small zip that inflates to gigabytes (a zip bomb).
+ENTRY_CAP = 256 * 1024 * 1024
+TOTAL_CAP = 1024 * 1024 * 1024 # across every entry one load reads: many entries just under ENTRY_CAP
+_READ_ERRORS = (zipfile.BadZipFile, zlib.error, EOFError, NotImplementedError, RuntimeError,
+ OverflowError, OSError)
+
+
+def open_archive(path: str) -> zipfile.ZipFile:
+ try:
+ return zipfile.ZipFile(path)
+ except (zipfile.BadZipFile, OSError, ValueError) as exc: # ValueError covers a bad UTF-8 entry name
+ raise FormatError("%s: not a readable zip archive (%s)" % (path, exc))
+
+
+def read_entry(archive: zipfile.ZipFile, info: zipfile.ZipInfo, cap: Optional[int] = None) -> bytes:
+ """The entry's bytes, read at most `cap` (default ENTRY_CAP) + 1 bytes far, so a lying size
+ field cannot make it allocate more."""
+ limit = ENTRY_CAP if cap is None else cap
+ try:
+ with archive.open(info) as stream:
+ data = stream.read(limit + 1)
+ except _READ_ERRORS as exc:
+ raise FormatError("entry %s could not be read (%s: %s)" % (info.filename, type(exc).__name__, exc))
+ if len(data) > limit:
+ raise FormatError("entry %s inflates past %d bytes; not read" % (info.filename, limit))
+ return data
+
+
+def entry_name(info: zipfile.ZipInfo) -> str:
+ """Entry name as text: the zip flag says UTF-8, otherwise cp437, with a GBK retry."""
+ raw = info.filename.encode("utf-8") if info.flag_bits & 0x800 else info.filename.encode("cp437")
+ try:
+ name = raw.decode("utf-8")
+ except UnicodeDecodeError:
+ name = raw.decode("gbk", "replace")
+ return name.replace("\\", "/").rsplit("/", 1)[-1]
diff --git a/skills/other-platforms/SKILL.md b/skills/other-platforms/SKILL.md
index 96efa71..e34394d 100644
--- a/skills/other-platforms/SKILL.md
+++ b/skills/other-platforms/SKILL.md
@@ -7,8 +7,10 @@ description: >-
block export, whole-project Unicode .csv, .plc/.ple, Bulk Data Manager) plus ABB
Automation Builder (AC500) and 800xA Control Builder M, and Rockwell Studio 5000 Logix
Designer (L5X/L5K, RLL neutral text XIC/XIO/OTE/TON, v36 IEC mnemonic renames, AOIs,
- FBD_TIMER in ST). Use when working with any of these tools or files. For Siemens and
- Schneider use their dedicated skills.
+ FBD_TIMER in ST). Also reading a CODESYS/Machine Expert binary `.project` without the IDE
+ (ST text, object inventory, build identity with `info`, comparing two versions with `diff`,
+ where a name is used with `xref`). Use when working with any of these tools or files. For
+ Siemens and Schneider use their dedicated skills.
---
# Other platforms: Beckhoff, CODESYS, ABB, Rockwell
@@ -20,6 +22,7 @@ differences from IEC 61131-3, and the traps.
|---|---|---|---|
| Beckhoff TwinCAT 3 | 3.1 Build 4026; PLC++ announced (plain-text files, "almost fully compliant" with 61131-3 Ed.4), GA unverified | `.TcPOU` / `.TcDUT` / `.TcGVL` / `.TcIO` with **ST** bodies only | [references/beckhoff-twincat.md](references/beckhoff-twincat.md) |
| CODESYS V3.5 | SP22 (Patch 2); SP23 planned Q1 2027 | PLCopenXML (`tc6_0200`), `.export` (native), File-Based Storage `*.st` (preview) | [references/codesys.md](references/codesys.md) |
+| CODESYS / Machine Expert `.project` (binary, no IDE) | undocumented format, reverse-engineered | **read only**: ST text and inventory by script; graphical wiring is a lead, not a fact | [references/codesys-project-files.md](references/codesys-project-files.md) |
| ABB Freelance (DCS) | Freelance 2024 (announced 2024-10-02); manuals read: 2019 | read the `.csv` export; bulk-edit via BDM Excel; **never hand-edit `.prt`/`.csv` for re-import without a backup** | [references/abb.md](references/abb.md) |
| ABB AC500 (Automation Builder) | CODESYS-based | PLCopenXML, `.export` (same-version only) | [references/abb.md](references/abb.md) |
| Rockwell Studio 5000 | v38 (Sept 2025) | L5X component exports, RLL neutral text, L5K | [references/rockwell-logix.md](references/rockwell-logix.md) |
@@ -38,6 +41,25 @@ differences from IEC 61131-3, and the traps.
`E_X.Member`.
- Logix has no qualifier: tags are controller- or program-scoped.
+## Reading a CODESYS `.project` without the IDE
+
+A repository that commits only the binary `.project` (CODESYS 3.5, Machine Expert) is not
+opaque to the plugin. The format is **undocumented**; the reader is reverse-engineered and
+checked on Machine Expert V2.x files only.
+
+- `python3 ${CLAUDE_PLUGIN_ROOT}/scripts/codesys_project.py list ` inventories
+ the objects; `st ` prints a POU's declaration and implementation;
+ `dump-st ` writes every ST POU and GVL/DUT declaration; `graph `
+ lists the element records of an LD/FBD POU; `info ` prints each application's
+ CodeId / DataId and the last save time (a lead to which saved project a PLC runs; the IDE decides);
+ `diff []` compares two versions; `xref ` lists every
+ use of a name (heuristic, by name only).
+- **Prefer a PLCopen XML export from the IDE when one can be had.** Use the reader for ST
+ text, inventories and as a lead, and **verify graphical wiring in the IDE** before acting
+ on `graph`: wiring is not decoded.
+- Details, what is decoded and what is a heuristic:
+ [references/codesys-project-files.md](references/codesys-project-files.md).
+
Validate: `python3 ${CLAUDE_PLUGIN_ROOT}/scripts/plc_validate.py `.
- TwinCAT rules: TC0xx.
diff --git a/skills/other-platforms/references/codesys-project-files.md b/skills/other-platforms/references/codesys-project-files.md
new file mode 100644
index 0000000..33ad3cf
--- /dev/null
+++ b/skills/other-platforms/references/codesys-project-files.md
@@ -0,0 +1,237 @@
+# Reading a CODESYS `.project` without the IDE
+
+For a CODESYS 3.5 or EcoStruxure Machine Expert `.project` that is the only copy of a
+program (a repository that commits the binary and nothing else). The plugin ships a
+reader, `scripts/codesys_project.py`, standard library only, Python 3.9+.
+
+## Status: undocumented and reverse-engineered
+
+**There is no vendor documentation for this format.** The layout below was worked out by
+looking at real project files. Nothing in it is confirmed by CODESYS or Schneider Electric,
+and a newer IDE may change it. Labels used on this page:
+
+- **Observed**: seen in the files examined, and implemented that way.
+- **Heuristic**: an interpretation of observed bytes, not confirmed against the IDE.
+- **Not decoded**: the reader does not try.
+
+Files examined (**observed**): three Machine Expert V2.x `.project` files of one application
+lineage, each holding Structured Text and LD/FBD POUs, saved by an IDE built on CODESYS
+3.5.21.50 (that version string occurs in the files' `.auxiliary` entries; the Machine Expert
+version itself is the maintainer's statement, not read from the files). **Nothing else was
+checked**: other CODESYS vendors (Beckhoff, WAGO, ABB), other service packs, CFC, SFC, IL,
+methods, actions, properties, visualisations and encrypted or compiled-library files are
+**unverified**. When the user's version differs, say so and ask for a PLCopen XML export.
+
+## Safe workflow
+
+1. **Prefer a PLCopen XML export from the IDE** (or the scriptengine on a machine that has
+ the IDE; formats and trade-offs in [codesys.md](codesys.md)). Connections between
+ graphical elements are explicit there. The reader cannot give you that.
+2. If only the `.project` exists, use the reader for **ST text, the object inventory, names,
+ declarations and as a lead** for graphical POUs.
+3. **Verify graphical wiring in the IDE** before acting on anything from `graph`. The reader
+ lists which elements a network contains; it does not say which output feeds which input.
+4. Never write back into a `.project`. The reader is read-only, and the format is not
+ understood well enough to write.
+
+## Commands
+
+```text
+python3 ${CLAUDE_PLUGIN_ROOT}/scripts/codesys_project.py list PROJECT [--json] [--kind KIND]
+python3 ${CLAUDE_PLUGIN_ROOT}/scripts/codesys_project.py st PROJECT NAME_OR_GUID [--decl | --impl]
+python3 ${CLAUDE_PLUGIN_ROOT}/scripts/codesys_project.py dump-st PROJECT OUTDIR
+python3 ${CLAUDE_PLUGIN_ROOT}/scripts/codesys_project.py graph PROJECT NAME_OR_GUID [--json]
+python3 ${CLAUDE_PLUGIN_ROOT}/scripts/codesys_project.py info PROJECT [--json]
+python3 ${CLAUDE_PLUGIN_ROOT}/scripts/codesys_project.py diff OLD NEW [NAME_OR_GUID] [--json]
+python3 ${CLAUDE_PLUGIN_ROOT}/scripts/codesys_project.py xref PROJECT NAME [--json]
+```
+
+| Command | Output |
+|---|---|
+| `list` | every object: guid, kind, subtype, name, ancestor names. `--kind` is one of `st`, `graphical`, `gvl`, `dut`, `other`, `unreadable` |
+| `st` | declaration and implementation text of an ST POU. For a graphical POU, a GVL or a DUT: the declaration only. `--decl` / `--impl` print one part raw |
+| `dump-st` | every ST POU as `_.decl.st` and `.impl.st`, and every GVL and DUT as `.decl.st`. Text is written byte for byte (CRLF kept) |
+| `graph` | the element records and network comments of an LD/FBD POU, as text or `--json` |
+| `info` | per device application: CodeId and DataId (the build identity), and the project's last save time. See [Which saved project matches a running PLC](#which-saved-project-matches-a-running-plc) |
+| `diff` | with a POU name or guid: the unified diff of its declaration and implementation (ST) or of its decoded graph (LD/FBD) between two project files. Without one: the objects added, removed and changed. See [Comparing two versions](#comparing-two-versions) |
+| `xref` | every ST and LD/FBD use of a variable, instance or type name, with a read / write role where one can be told. **Heuristic**, see [Finding where a name is used](#finding-where-a-name-is-used) |
+
+- **Names are not unique.** Two devices can each have an `Application`, and a POU can
+ exist once per device. `st` and `graph` refuse an ambiguous name (exit 2) and list the
+ candidates with their ancestors; give the guid, or the first 8 or more characters of it.
+ Names and guids match in any case. `diff OLD NEW ` takes a renamed POU's old or new
+ name: it is resolved where it exists and found in the other version by guid.
+- **`dump-st` file names** start with the root-most ancestor (the device), which is a
+ heuristic. Names that collide, including by case, all get `_`, so no
+ file is overwritten. Names come from the file and are made safe as one path component.
+- **Exit codes:** 0 clean; 1 the result was printed but some objects had problems (listed on
+ stderr, one `PROBLEM ` line each) or a decoder note was raised; 2 usage error,
+ unreadable file, unknown or ambiguous name, or the object has the wrong kind for the command.
+ `diff` exits 0 when the two projects differ: a difference is its result, not a problem.
+- A damaged object is reported and skipped; it does not stop the other objects from being read.
+ A zip entry that fails its CRC, does not inflate, is encrypted, or inflates past 256 MiB is
+ that object's problem; the same failure on the string table makes the file unreadable (exit 2).
+ `info` refuses an XML entry that declares a DOCTYPE or an ENTITY, whatever its encoding.
+
+## What is decoded
+
+| Item | Status | How |
+|---|---|---|
+| Zip container, shared string table | **Observed** | `.project` is an ordinary zip. `__shared_data_storage_string_table__.auxiliary` is a varint count, then (index, byte length, UTF-8 bytes) entries. A count that does not match, a duplicate index or an entry past the end of the file is an error |
+| Object guid, parent guid, name | **Observed** | each object is `.meta` plus `.object`. The `.meta` is a varint stream; mapped through the string table, the object's own guid is followed by its parent guid and its name |
+| Object header | **Observed** | 20 bytes: `02 20 09 28`, 12 zero bytes, then a little-endian uint32 equal to the number of bytes that follow. All objects of the three files satisfied it. A violation marks the object `unreadable` |
+| ST declaration and implementation text | **Observed** | the object holds string-table indices, not text. A text slot is ` e5 02 `, marker `41` for the declaration and `5b` for the implementation; the reference string is `_Decl_LineIds` or `_Impl_LineIds`. A marker whose reference does not match is treated as data. 32 of 32 ST files of one project were byte-identical to an earlier extraction that located the text by fixed varint positions instead (a different rule, same file) |
+| Network comments, element name and type, box type, instance and header id, pin names, modifier byte | **Observed** | see Graphical objects |
+| Kind (`st`, `graphical`, `gvl`, `dut`, `other`) | **Heuristic** | from the payload: a network record means graphical; a declaration slot and an implementation slot mean ST; a declaration slot only is a GVL (`VAR…`) or a DUT (`TYPE`); anything else is `other`. Vendor type guids are not used because their position in the `.meta` varied |
+| `subtype` | **Heuristic** | the first keyword of the declaration after comments and pragmas: `PROGRAM`, `FUNCTION_BLOCK`, `FUNCTION`, `METHOD`, ... |
+| Device, application | **Heuristic** | device is the root-most ancestor; application is the node under a node named `Plc Logic` (case-insensitive) |
+| Build identity: CodeId, DataId, device model and firmware version | **Observed** | `devicesmetainformation.auxiliary` is plain XML: a `` per controller with `Name`, `DeviceType`, `DeviceId`, `DeviceVersion`, `DeviceName` and one `` with `Name`, `CodeId`, `DataId` (`System.Guid`). In all three files the id is a guid whose first group varies and whose other groups are zero |
+| Last save time | **Observed** | `engineeringtoolsversions.auxiliary` holds a `` of entries with `LastTimeSaved` (`YYYY-MM-DD-HH-MM-SS`, the saving machine's clock) and `SaveCounter`. The newest `LastTimeSaved` equalled the zip entries' own save time to within seconds in all three files. It is a **save**, not a compile |
+
+## Graphical objects
+
+**Observed:** a graphical (LD/FBD) object holds its declaration in a text slot near the end,
+a language string in its header (only `Ld` was seen, on 12 of 12 graphical objects; the
+reader prints it and does not tell LD from FBD), and one record per network. A network record
+starts `0f e1 02` or `0f e7 02` (both are networks with a comment; what separates the two
+tags is **not decoded**).
+
+`graph` scans each network and lists, in file order:
+
+| Record | Decoded fields | Heuristic fields |
+|---|---|---|
+| element | name, type, modifier byte (raw) | `IN`/`OUT` (the byte before the record is `69` for an input), `coil` (an output right after a coil-count record), the modifier meaning |
+| box | type, instance name, header id | nesting depth (a box spans from its header id H to the next 8-byte id H+1) |
+| coils | count byte | |
+| pins | pin names | |
+| empty | raw bytes, id | what an empty slot means. An empty name is string-table index 39 (`""` in every project examined; **observed**, the reader does not look it up) |
+| parallel_marker | value byte | what it means (**unverified**) |
+| unknown | the reason and 16 raw bytes | a record that was recognised but could not be read (a name index not in the string table, a record that runs past the end) |
+
+Modifier meanings are **unverified** against the IDE. The reader always shows the raw code
+and adds the labels `negated` (1), `set` (2) and `reset` (3) as a heuristic. What was
+observed in the three files: `2` and `3` only on coil outputs, `1` on inputs and on coils,
+`16` and `32` only on inputs. The labels fit the usual LD coil and contact kinds, but no
+manual or IDE check confirms them, and `16` and `32` have no label. An `IN` is either an LD
+contact or an FBD input; the file does not say which.
+
+**Not decoded: the wiring.** `ids` are 8-byte little-endian values found by scanning, and
+the filler id `733` is dropped. They are a **heuristic**: a one-byte id (below 256) can come
+out shifted by a byte (value times 256 plus the byte before it). Use them to compare within
+one object, never as an absolute id, and never to claim that one element feeds another.
+Bytes that match no recognised record are skipped without a note, so a clean `graph` run
+does not mean every byte was understood. The notes list only records that were recognised
+and could not be read.
+
+## Comparing two versions
+
+`diff OLD NEW POU` compares one POU between two project files; `diff OLD NEW` lists what
+changed in the whole project. Objects are matched by **guid**, which stayed the same across
+the saved versions of one project examined (**observed**: in three pairs of versions compared,
+no object of the older file was missing by guid from the newer one); a renamed object keeps its guid and is listed as
+`renamed from`, a moved one as `moved from`.
+
+- **ST and declarations:** the text is compared line by line, line endings ignored.
+- **LD/FBD:** the decoded graph is compared as text with everything that changes on a re-save
+ left out: byte offsets, element ids, box header ids, the id and bytes of an empty slot, the
+ raw bytes of an unreadable record. What stays in: element names and types, **constants**,
+ the modifier byte, network comments, box type and instance, box nesting and the coil count.
+ A constant that changes from 500 to 600 shows. A network inserted in the middle shows as an
+ insertion, not as every later network changing.
+- **Not compared, so not visible:** the **wiring** (the reader does not decode it), and bytes
+ the graph decoder does not recognise. "No differences" means no difference in the fields above,
+ never that the two POUs are identical. Where the wiring matters, use the IDE's own compare.
+- Objects of kind `other` (devices, tasks, I/O mappings) are counted as "not compared", not
+ diffed. The project-level summary lists what was added, removed and changed, with the sections
+ that differ (`declaration`, `implementation`, `graph`).
+- The modifier labels (`~negated`, `~set`, `~reset`) and `IN`/`OUT` are the same **heuristics**
+ as in `graph`; a changed label means a changed modifier byte, not necessarily a changed meaning.
+- **Checked on real files** (not part of this repository): the graph diff of one POU between two
+ saved versions of one application found the same element, comment and box changes as an
+ earlier text diff of two decoded dumps. It also showed two four-digit constant changes that
+ the earlier diff had normalised away together with the ids, and it reported no difference for
+ a POU whose bytes were identical in two versions. Between two versions of that application the
+ six ST POUs that `diff` listed as changed were exactly the six whose `dump-st` files differ
+ byte for byte. No real graphical object was
+ found whose bytes differed while its normalised graph did not, so the id and offset
+ normalisation is proven by the synthetic tests only.
+
+## Finding where a name is used
+
+`xref PROJECT NAME` lists every use of a variable, function block instance or type name in ST
+text, declarations and decoded LD/FBD elements. **Everything it prints is a heuristic.**
+
+- **By name only.** Matching is case-insensitive (IEC 61131-3 is) and has no scope resolution: a
+ local variable and a global with the same name both match, and `x.Foo` matches the query `Foo`
+ as well as `Foo.x`. A dotted query (`Gvl.Foo`) matches paths with those segments in a row.
+- **Roles in ST:** `write` is the target before `:=` outside a call, or the variable after `=>`;
+ `read` is any other use (an array index of a target is a read); `call` is a name followed by
+ `(`; `param` is the formal name before `:=` or `=>` inside a call; `arg` is the value after
+ `name :=` inside a call. **`arg` is not a read:** whether the callee writes it depends on the
+ parameter's declaration (`VAR_IN_OUT`, `VAR_OUTPUT`), which is not read.
+- **Roles in declarations:** `declared`, `type` (after the `:`, and a header's return type or `TYPE` definition), `read` (an initial value). A header (`FUNCTION F : BOOL`, `TYPE T : STRUCT`) declares its own name.
+- **LD/FBD:** `write` for an element the decoder labels OUT, `read` for IN (both resting on the
+ heuristic direction), `read` for names inside an index, `type` for a box type, `call` for a
+ box instance. A constant is not a name. Wiring is not decoded.
+- **What it cannot see:** objects of kind `other` (the device tree: a variable mapped to an I/O
+ channel there is not found), graphical records the decoder could not read (the output says
+ how many), and names built at run time (pointers, `ADR` arithmetic). A name that does not
+ appear in the output may still be used; treat "no hits" as "none found", not as "unused".
+- Comments, pragmas and string literals are skipped; nested block comments are handled.
+
+## Which saved project matches a running PLC
+
+When several saved copies of a program exist and one controller runs one of them, `info`
+prints what the IDE uses to tell: the **CodeId** and **DataId** of each device's application,
+as stored in the project. The IDE compares them with the controller's (**not verified here**:
+the vendor does not document the comparison).
+
+1. In the IDE, online with the controller, read the application's CodeId (the menu differs
+ between versions; which one shows it was not checked here).
+2. Run `info` on each candidate `.project`. A candidate whose CodeId equals the controller's is
+ the build that runs. Equal DataIds across two versions suggest the retained and persistent
+ variables did not change (**inferred**, one lineage; what DataId encodes is not documented).
+3. **Both byte orders are printed.** The boot image the IDE writes beside the project
+ (`.bootinfo_guids`, not part of the zip) stores the id with its first four bytes reversed
+ (**observed**: of 8 ids, the CodeId and DataId of two applications in each of two projects, 7
+ occur in that file in the reversed order and none in the plain order). Which order the IDE
+ *displays* was **not checked**: compare against both before concluding "no match".
+4. **A mismatch is not proof.** In the same check one CodeId from the project metadata occurred
+ in the boot image beside it in neither order, and the cause is unknown. The IDE's comparison
+ with the real controller is what decides, never this reader.
+5. **If none matches,** the controller may run a build that was never saved (an edit downloaded
+ and the project closed unsaved: a possibility, not observed here). Say so rather than naming
+ the nearest candidate.
+
+**There is no compile time in the project.** `info` prints the last **save** time, which can
+be earlier or later than the build that runs. The compile time is written by the IDE next to the
+project file (**unverified**: `.compileinfo` and `.bootinfo` files beside the project were seen,
+and are not part of the zip and not decoded here), and by the IDE's build messages. When the identity cannot be read off the
+controller, match the build to **logged data** instead: find when a logged signal's behaviour
+changed, take the compile time of each candidate (**unverified**: that the `.compileinfo` file's
+modification time is the compile time was not checked; this reader does not read the file), and use
+`diff` between consecutive candidates to see which POU changes could explain the change. Name the
+evidence tier: a timestamp match is a coincidence candidate until a CodeId confirms it.
+
+## Checking a decode
+
+`graph` output is a lead. To use it:
+
+1. Compare the element list and the network comments with the network in the IDE.
+2. If a PLCopen XML export exists, the explicit connections there take precedence.
+3. Treat a missing element as possible: an element whose record shape was not recognised is
+ skipped silently.
+
+## Sources and checks
+
+- **No vendor source exists** for the container, string table, `.meta`, `.object`, text
+ slots or graphical records. Everything above is **observed** on the files named in Status.
+- The reader was compared locally with earlier ad-hoc extraction scripts on the real files
+ (not part of this repository): the text of all 32 ST files of one project matched byte for
+ byte. For the 12 graphical objects of that project the reader found all 2189 element, box,
+ coil and empty-slot records of the earlier tree decoder (offset, kind, names, box id); 5
+ differ only in the label for modifier 32, and the reader found 35 more empty slots, which
+ the earlier tool's pattern missed.
+- The tests in `tests/test_codesys_project.py` and `tests/test_codesys_tools.py` use **synthetic** projects built from this
+ layout. They show the reader implements what is written here; they do not show that the
+ layout matches every file the IDE produces.
diff --git a/skills/other-platforms/references/codesys.md b/skills/other-platforms/references/codesys.md
index 09aa4e5..85eca8d 100644
--- a/skills/other-platforms/references/codesys.md
+++ b/skills/other-platforms/references/codesys.md
@@ -19,7 +19,7 @@ flavour and which CODESYS SP is underneath.** Libraries, devices and some menus
| Format | What it is |
|---|---|
-| `.project` / `.library` | a single compressed file; binary as far as Git is concerned |
+| `.project` / `.library` | a single compressed file; binary as far as Git is concerned. A `.project` can be read without the IDE by `scripts/codesys_project.py`: [codesys-project-files.md](codesys-project-files.md) (undocumented format, reverse-engineered) |
| `.compiled-library-v3` | an encrypted library with no source |
| `*.export` | native XML, "fully compatible" with the project format |
| `*.xml` PLCopenXML | namespace `tc6_0200`; methods, properties and folders live in `addData` (see the `exchange-formats` skill); "100% compatibility cannot be guaranteed" |
diff --git a/skills/schneider/SKILL.md b/skills/schneider/SKILL.md
index 5712322..9bf63e1 100644
--- a/skills/schneider/SKILL.md
+++ b/skills/schneider/SKILL.md
@@ -97,6 +97,11 @@ breakpoints**.
- [references/exchange-files.md](references/exchange-files.md): Machine Expert
PLCopenXML limits, Control Expert XEF/ZEF/XSY with a skeleton, Machine Expert – Basic.
+Machine Expert M241/M251/M262 programs are `.project` files (CODESYS 3.5). To read one
+without the IDE (ST text, inventory, graphical POUs as a lead), see
+[../other-platforms/references/codesys-project-files.md](../other-platforms/references/codesys-project-files.md):
+undocumented format, reverse-engineered, prefer a PLCopen XML export when you can get one.
+
Language syntax, pragmas and Static Analysis for Machine Expert are the CODESYS ones:
[../other-platforms/references/codesys.md](../other-platforms/references/codesys.md).
Validate XEF/XSY and PLCopenXML with
diff --git a/tests/codesys_synth.py b/tests/codesys_synth.py
new file mode 100644
index 0000000..d21fda5
--- /dev/null
+++ b/tests/codesys_synth.py
@@ -0,0 +1,312 @@
+"""Builders for SYNTHETIC CODESYS `.project` files, used by test_codesys_project.py.
+
+The encoder lives here and not in the reader on purpose: the tests check the reader against
+the layout it documents, written down a second time. All names and text are invented.
+"""
+
+from __future__ import annotations
+
+import struct
+import zipfile
+
+from codesysproj import project
+
+FILLER = 300 # string indices below this are filler, so real text gets two-byte varint indices
+MAGIC = bytes.fromhex("02200928")
+
+
+def vi(n: int) -> bytes:
+ out = bytearray()
+ while True:
+ byte, n = n & 0x7F, n >> 7
+ out.append(byte | 0x80 if n else byte)
+ if not n:
+ return bytes(out)
+
+
+def qword(n: int) -> bytes:
+ return struct.pack(" str:
+ return "%08x-0000-4000-8000-%012x" % (n, n)
+
+
+class Table:
+ def __init__(self):
+ self.texts = ["filler-%d" % i for i in range(FILLER)]
+ self.index = {}
+
+ def add(self, text: str) -> int:
+ if text not in self.index:
+ self.index[text] = len(self.texts)
+ self.texts.append(text)
+ return self.index[text]
+
+ def v(self, text: str) -> bytes:
+ return vi(self.add(text))
+
+ def dump(self, declared=None) -> bytes:
+ body = b"".join(vi(i) + vi(len(t.encode())) + t.encode() for i, t in enumerate(self.texts))
+ return vi(len(self.texts) if declared is None else declared) + body
+
+
+def make_object(payload: bytes) -> bytes:
+ return MAGIC + b"\x00" * 12 + struct.pack(" bytes:
+ head = bytes([2, 0x20, 9, 0x28]) + b"\x00" * 12
+ ids = table.v(own) * (2 if repeat_own else 1) + table.v(parent) + table.v(name)
+ return head + ids + b"\x00\x00" + table.v("7777") + b"\x80" * 11 # a broken tail varint
+
+
+def slot(table: Table, owner: str, text: str, impl: bool) -> bytes:
+ marker = b"\x5b\xe5\x02" if impl else b"\x41\xe5\x02"
+ ref = owner + ("_Impl_LineIds" if impl else "_Decl_LineIds")
+ return marker + table.v(text) + table.v(ref)
+
+
+def st_payload(table: Table, owner: str, decl: str, impl: str) -> bytes:
+ return (b"\x0f\xe6\x02\x48" + slot(table, owner, impl, True) + slot(table, owner, decl, False)
+ + b"\xd0\x01\x00\x00\x00")
+
+
+def decl_only_payload(table: Table, owner: str, decl: str) -> bytes:
+ return b"\x0f\x80\x01" + slot(table, owner, decl, False) + b"\x00\x00\x00"
+
+
+def network(table: Table, comment, items: bytes = b"", tag: bytes = b"\xe1\x02") -> bytes:
+ comment_bytes = b"\x00" if comment is None else table.v(comment)
+ return b"\x0f" + tag + b"\x00" * 5 + comment_bytes + b"\x27\x27\x00\x01" + items
+
+
+def graphical_payload(table: Table, owner: str, networks: bytes, decl: str, lang="Ld", pad=b"") -> bytes:
+ return (b"\x0f\xe6\x02\x48\x9c\x01\x27" + table.v(lang) + b"\x04" + pad + networks
+ + slot(table, owner, decl, False) + b"\xd0\x01\x00\x00\x00")
+
+
+def coils(count: int) -> bytes:
+ return b"\x0f\x6f\x6c" + bytes([count])
+
+
+def element(table: Table, name: str, typ: str, modifier: int, ident: int, is_input=False) -> bytes:
+ head = (b"\x0f\x69" if is_input else b"") + b"\xdf\x02"
+ return (head + table.v(name) + table.v(typ) + b"\x27\x27\x27\xdd\x02" + bytes([modifier])
+ + b"\x00" * 5 + b"\x01\x00\x00" + qword(ident))
+
+
+def box(table: Table, box_type: str, instance: str, header_id: int, tag=b"\xe0\x02") -> bytes:
+ inst = b"\xde\x02\x00" if not instance else b"\xdf\x02" + table.v(instance)
+ return (b"\x0f" + tag + table.v(box_type) + inst + b"\x27\x27\x27\x27\xdd\x02" + b"\x00" * 8
+ + b"\x01" + qword(header_id))
+
+
+def pins(table: Table, *names: str) -> bytes:
+ return bytes([0x6A, len(names), 0x0E]) + b"".join(table.v(n) for n in names)
+
+
+class Synthetic:
+ """Two devices that each hold an Application with a same-named POU, plus one object of every kind."""
+
+ def __init__(self):
+ self.t = Table()
+ self.files = {}
+ self.guids = {}
+ self._n = 0
+
+ def add(self, name: str, parent: str, payload: bytes = None, raw: bytes = None, meta=True, **kw) -> str:
+ self._n += 1
+ g = guid(self._n)
+ self.guids[name + str(self._n)] = g
+ self.files[g] = {}
+ if meta:
+ self.files[g]["meta"] = (parent, name, kw.get("repeat_own", False))
+ if raw is not None:
+ self.files[g]["object"] = raw
+ elif payload is not None:
+ self.files[g]["object"] = payload
+ return g
+
+ def write(self, path: str, declared=None, extra=None) -> None:
+ with zipfile.ZipFile(path, "w", zipfile.ZIP_DEFLATED) as z:
+ for name, data in (extra or {}).items():
+ z.writestr(name, data)
+ for g, parts in self.files.items():
+ if "meta" in parts:
+ parent, name, repeat = parts["meta"]
+ z.writestr(g + ".meta", make_meta(self.t, g, parent, name, repeat))
+ if "object" in parts:
+ obj = parts["object"]
+ z.writestr(g + ".object", obj(g) if callable(obj) else obj)
+ z.writestr(project.STRING_TABLE_NAME, self.t.dump(declared))
+
+
+def build_standard(path: str) -> dict:
+ """The project most tests read. Returns the guids by role."""
+ s = Synthetic()
+ t = s.t
+ null = guid(0)
+ ids = {}
+ for dev in ("DevA", "DevB"):
+ d = s.add(dev, null, make_object(b"\x0f\x01"))
+ pl = s.add("Plc Logic", d, make_object(b"\x0f\x02"))
+ app = s.add("Application", pl, make_object(b"\x0f\x03"))
+ ids[dev] = app
+ ids[dev + ".device"] = d
+ ids[dev + ".pou"] = s.add("Main", app, lambda g, dev=dev: make_object(
+ st_payload(t, g, "PROGRAM Main\nVAR\n\tx: INT;\nEND_VAR\r\n", "x := x + 1; // %s\r\n" % dev)))
+ ids["gvl"] = s.add("Globals", ids["DevA"], lambda g: make_object(
+ decl_only_payload(t, g, "VAR_GLOBAL\n\tcount: INT;\nEND_VAR")))
+ ids["dut"] = s.add("Point", ids["DevA"], lambda g: make_object(decl_only_payload(t, g, "TYPE Point :\nSTRUCT\nEND_STRUCT\nEND_TYPE")))
+ ids["pragma"] = s.add("Pragma", ids["DevA"], lambda g: make_object(
+ st_payload(t, g, "{attribute 'hide'}\n// note\n(* block *)\nFUNCTION_BLOCK Pragma\nVAR\nEND_VAR", "")))
+ items = (coils(1) + element(t, "Lamp", "BOOL", 2, 5111)
+ + element(t, "Start", "BOOL", 1, 5230, is_input=True)
+ + box(t, "TON", "Timer1", 6034) + element(t, "Preset", "TIME", 0, 5340, is_input=True)
+ + qword(6035) + pins(t, "IN", "PT") + b"\x9a\x01\x42\x00\x00\x00\x00" + qword(7451))
+ nets = network(t, "start the lamp", items) + network(t, None, element(t, "Y", "INT", 0, 5450), tag=b"\xe7\x02")
+ ids["graph"] = s.add("Ladder", ids["DevA"], lambda g: make_object(
+ graphical_payload(t, g, nets, "PROGRAM Ladder\nVAR\nEND_VAR", "Ld")))
+ ids["other"] = s.add("Tasks", ids["DevA.device"], make_object(b"\x0f\x01\x02\x03"))
+ # entries that are not a `.meta` / `.object` pair are ignored
+ s.write(path, extra={"notes.meta": b"x", "readme.object": b"x", guid(4242) + ".txt": b"x"})
+ return ids
+
+
+# ---- two versions of one project, a build-identity entry, and a project made for xref ----
+
+def devices_meta_xml(devices) -> bytes:
+ """`devicesmetainformation.auxiliary`: [(name, [(app name, code id, data id), ...]), ...]."""
+ out = ['']
+ for name, apps in devices:
+ out.append('%s' % name)
+ out.append('4096')
+ out.append('0000 0001')
+ out.append('1.2.3.4')
+ out.append('Model-%s' % name)
+ for app, code, data in apps:
+ out.append('%s' % app)
+ out.append('%s' % code)
+ out.append('%s' % data)
+ out.append("")
+ out.append("")
+ return "".join(out).encode()
+
+
+def tools_versions_xml(saves, product="Test IDE", version="2.6") -> bytes:
+ """`engineeringtoolsversions.auxiliary`: saves = [(first, last, save counter, core version), ...]."""
+ entries = "".join(
+ '' % (core, counter, first, last) for first, last, counter, core in saves)
+ return (''
+ '%s'
+ % (version, product, entries)).encode()
+
+
+def _ladder(t: Table, version: int, shifted: bool) -> bytes:
+ """Version 1 and 2 of a ladder with one constant, one modifier, one comment and one element changed."""
+ base = 100 if version == 1 else 9000 # element ids: all different between versions
+ const = "500" if version == 1 else "600"
+ lamp_mod = 2 if version == 1 else 3
+ comment = "start the lamp" if version == 1 else "start the lamp slowly"
+ items = (coils(1) + element(t, "Lamp", "BOOL", lamp_mod, base + 11)
+ + element(t, "Start", "BOOL", 1, base + 12, is_input=True)
+ + box(t, "TON", "Timer1", base + 20) + element(t, const, "", 0, base + 13, is_input=True)
+ + qword(base + 21) + pins(t, "IN", "PT"))
+ nets = network(t, comment, items)
+ nets += network(t, None, element(t, "Y", "INT", 0, base + 40), tag=b"\xe7\x02")
+ if version == 2:
+ nets += network(t, "added later", element(t, "Extra", "INT", 0, base + 50))
+ return nets
+
+
+def _shifted(t: Table, version: int) -> bytes:
+ """The same content in both versions; only the element ids and the byte offsets differ."""
+ base = 200 if version == 1 else 7000
+ items = (coils(1) + element(t, "Out1", "BOOL", 0, base + 1) + element(t, "In1", "BOOL", 1, base + 2, is_input=True)
+ + b"\x9a\x01" + bytes([0x40 + version]) + b"\x00\x00\x00\x00" + qword(base + 3)) # an empty slot
+ return network(t, "same network", items)
+
+
+def build_version(path: str, version: int, meta=True) -> dict:
+ """Version 1 or 2 of one project. Objects present in both keep their guid.
+
+ Changes 1 -> 2: `Edited` (declaration and implementation), `Ladder` (constant, modifier, network
+ comment, one network added), `Renamed` (name only), `Globals` (declaration), `Tasks` (opaque bytes).
+ Unchanged: `Same`, `Shifted` (ids and offsets differ, content does not). `Gone` exists only in 1,
+ `Fresh` only in 2.
+ """
+ s = Synthetic()
+ t = s.t
+ v = version
+ null = guid(0)
+ dev = s.add("DevA", null, make_object(b"\x0f\x01"))
+ pl = s.add("Plc Logic", dev, make_object(b"\x0f\x02"))
+ app = s.add("Application", pl, make_object(b"\x0f\x03"))
+ ids = {"device": dev, "app": app}
+
+ def st(name, decl, impl):
+ return s.add(name, app, lambda g: make_object(st_payload(t, g, decl, impl)))
+
+ ids["same"] = st("Same", "PROGRAM Same\nVAR\nEND_VAR", "a := 1;\r\n")
+ ids["edited"] = st("Edited", "PROGRAM Edited\nVAR\n\tx: INT;\n%sEND_VAR" % ("\ty: INT;\n" if v == 2 else ""),
+ "x := x + %d;\r\n" % v)
+ ids["ladder"] = s.add("Ladder", app, lambda g: make_object(
+ graphical_payload(t, g, _ladder(t, v, False), "PROGRAM Ladder\nVAR\nEND_VAR")))
+ ids["shifted"] = s.add("Shifted", app, lambda g: make_object(
+ graphical_payload(t, g, _shifted(t, v), "PROGRAM Shifted\nVAR\nEND_VAR", pad=b"\x00" * (3 * v))))
+ ids["renamed"] = st("OldName" if v == 1 else "NewName", "PROGRAM R\nVAR\nEND_VAR", "r := 1;\r\n")
+ ids["globals"] = s.add("Globals", app, lambda g: make_object(
+ decl_only_payload(t, g, "VAR_GLOBAL\n\tcount: INT;\n%sEND_VAR" % ("\tmore: INT;\n" if v == 2 else ""))))
+ ids["tasks"] = s.add("Tasks", app, make_object(b"\x0f\x01\x02" + bytes([v])))
+ if v == 1:
+ ids["gone"] = st("Gone", "PROGRAM Gone\nVAR\nEND_VAR", "g := 1;\r\n")
+ s._n += 1 # the guid `Fresh` takes in version 2
+ else:
+ s._n += 1 # the guid `Gone` had in version 1
+ ids["fresh"] = st("Fresh", "PROGRAM Fresh\nVAR\nEND_VAR", "f := 1;\r\n")
+ extra = {}
+ if meta:
+ code, data = ("aaaa%04x-0000-0000-0000-000000000000" % v, "bbbb%04x-0000-0000-0000-000000000000" % v)
+ extra["devicesmetainformation.auxiliary"] = devices_meta_xml([("DevA", [("Application", code, data)])])
+ extra["engineeringtoolsversions.auxiliary"] = tools_versions_xml(
+ [("2020-01-01-00-00-00", "2020-02-02-12-00-00", 5, "1.0.0.0"),
+ ("2020-03-01-00-00-00", "2020-0%d-15-08-30-45" % (v + 3), 9, "2.0.0.0")])
+ s.write(path, extra=extra)
+ return ids
+
+
+def build_xref(path: str) -> dict:
+ """ST, graphical and declaration-only objects that mention `counter` in every way xref classifies."""
+ s = Synthetic()
+ t = s.t
+ null = guid(0)
+ dev = s.add("Dev", null, make_object(b"\x0f\x01"))
+ app = s.add("Application", s.add("Plc Logic", dev, make_object(b"\x0f\x02")), make_object(b"\x0f\x03"))
+ ids = {}
+ ids["gvl"] = s.add("Gvl", app, lambda g: make_object(
+ decl_only_payload(
+ t, g, "VAR_GLOBAL RETAIN counter : INT;\n\tother, spare : INT;\n\tlone AT %MX0.0 : BOOL;\nEND_VAR")))
+ impl = ("// counter in a comment\n"
+ "(* counter := 9; *)\n"
+ "counter := counter + 1;\n"
+ "Gvl.counter := 2;\n"
+ "arr[counter] := 0;\n"
+ "msg := 'counter in a string';\n"
+ "Timer1(IN := COUNTER > 3, PT := T#5s, Q => counter);\n"
+ "IF other = 1 THEN counter_total := 0; END_IF\n"
+ "FOR counter := 0 TO 9 DO other := other + 1; END_FOR\n"
+ "Fb.Method(counter);\n"
+ "Gvl.other := 3;\n")
+ ids["writer"] = s.add("Writer", app, lambda g: make_object(st_payload(
+ t, g, "PROGRAM Writer\nVAR\n\tlocal : INT := Gvl.counter;\n\ttimer1 : TON;\nEND_VAR", impl)))
+ items = (coils(1) + element(t, "Gvl.counter", "BOOL", 0, 5111) + element(t, "other", "BOOL", 0, 5230, is_input=True)
+ + box(t, "TON", "Timer1", 6034) + element(t, "arr[counter]", "INT", 0, 5340, is_input=True)
+ + element(t, "5", "", 0, 5341, is_input=True) + qword(6035) + pins(t, "IN", "PT"))
+ nets = network(t, "first", items) + network(t, None, element(t, "arr[idx]", "INT", 0, 5450), tag=b"\xe7\x02")
+ ids["lad"] = s.add("Lad", app, lambda g: make_object(
+ graphical_payload(t, g, nets, "PROGRAM Lad\nVAR\n\tcounter : INT;\nEND_VAR")))
+ ids["tasks"] = s.add("Tasks", dev, make_object(b"\x0f\x01\x02\x03"))
+ s.write(path)
+ return ids
diff --git a/tests/test_codesys_hardening.py b/tests/test_codesys_hardening.py
new file mode 100644
index 0000000..f2ee4df
--- /dev/null
+++ b/tests/test_codesys_hardening.py
@@ -0,0 +1,432 @@
+"""Hostile and damaged input, and the CLI contracts around it, tested on SYNTHETIC projects only.
+
+A reader handed a file it did not write must answer with a message and an exit code, never a
+traceback or a hang: damaged or encrypted zip entries, an entry that inflates past the cap, an
+XML entity declaration in any encoding, and declarations that make a regex backtrack.
+"""
+
+from __future__ import annotations
+
+import dataclasses
+import os
+import struct
+import subprocess
+import sys
+import tempfile
+import time
+import unittest
+import zipfile
+from pathlib import Path
+from unittest import mock
+
+HERE = Path(__file__).resolve().parent
+sys.path.insert(0, str(HERE.parent / "scripts"))
+sys.path.insert(0, str(HERE))
+
+import codesys_project # noqa: E402
+from codesysproj import buildinfo, diff, project, stscan, xref, zipread # noqa: E402
+from codesys_synth import ( # noqa: E402
+ Synthetic, build_version, decl_only_payload, devices_meta_xml, guid, make_object, st_payload,
+)
+from test_codesys_tools import run_cli # noqa: E402
+
+
+def _entry_data_offset(path: str, name: str) -> int:
+ with zipfile.ZipFile(path) as z:
+ info = z.getinfo(name)
+ with open(path, "rb") as f:
+ f.seek(info.header_offset)
+ header = f.read(30)
+ name_len, extra_len = struct.unpack_from(" None:
+ """Flip bytes inside the entry's compressed data: the inflate or the CRC check then fails."""
+ start = _entry_data_offset(path, name)
+ with open(path, "r+b") as f:
+ f.seek(start + 2)
+ chunk = f.read(4)
+ f.seek(start + 2)
+ f.write(bytes(b ^ 0xFF for b in chunk))
+
+
+def _mark_encrypted(path: str, name: str) -> None:
+ """Set the 'encrypted' flag of the entry's central-directory record."""
+ data = bytearray(Path(path).read_bytes())
+ at = data.index(b"PK\x01\x02")
+ while True:
+ name_len = struct.unpack_from("&e;'
+ for data in (doc.replace("UTF-16", "utf-8").encode("utf-8"), doc.encode("utf-16")):
+ info = self.info_of(data)
+ self.assertEqual(info.devices, [])
+ self.assertIn("DOCTYPE or ENTITY", " ".join(info.notes))
+
+ def test_a_plain_document_is_still_read(self):
+ info = self.info_of(devices_meta_xml([("Ctrl", [("App", "ab000000-0000-0000-0000-000000000000", None)])]))
+ self.assertEqual([d.name for d in info.devices], ["Ctrl"])
+ self.assertEqual(info.notes, [])
+
+ def test_a_devices_entry_with_no_device_is_a_note(self):
+ info = self.info_of(b'')
+ self.assertIn("no element", " ".join(info.notes))
+
+ def test_a_non_ascii_digit_save_counter_is_not_a_number(self):
+ root = buildinfo.ET.fromstring('')
+ info = buildinfo.ProjectInfo()
+ buildinfo._newest_save(root, info)
+ self.assertEqual((info.last_saved, info.save_counter), ("2020-01-02 03:04:05", None))
+
+
+class Versions(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ cls.tmp = tempfile.TemporaryDirectory()
+ cls.old = os.path.join(cls.tmp.name, "v1.project")
+ cls.new = os.path.join(cls.tmp.name, "v2.project")
+ cls.ids = build_version(cls.old, 1)
+ build_version(cls.new, 2)
+
+ @classmethod
+ def tearDownClass(cls):
+ cls.tmp.cleanup()
+
+ def test_a_renamed_pou_is_found_by_its_old_and_its_new_name(self):
+ for name in ("OldName", "NewName"):
+ code, out, err = run_cli("diff", self.old, self.new, name)
+ self.assertEqual((code, err), (0, ""), name)
+ self.assertIn("# renamed: 'OldName' -> 'NewName'", out)
+
+ def test_a_pou_missing_on_one_side_names_that_side(self):
+ self.assertRegex(run_cli("diff", self.old, self.new, "Gone")[2], r"is not in NEW .*: it was removed")
+ self.assertRegex(run_cli("diff", self.old, self.new, "Fresh")[2], r"is not in OLD .*: it was added")
+ self.assertIn("not found in OLD", run_cli("diff", self.old, self.new, "Nowhere")[2])
+
+ def test_the_headers_carry_each_versions_own_name(self):
+ old_p, new_p = project.load_project(self.old), project.load_project(self.new)
+ a = old_p.objects[self.ids["renamed"]]
+ b = dataclasses.replace(new_p.objects[self.ids["renamed"]], implementation="changed;")
+ lines = diff.object_diff(old_p, a, new_p, b).sections[0].lines
+ self.assertEqual(lines[:2], ["--- OLD OldName implementation", "+++ NEW NewName implementation"])
+
+
+class ProblemsAndNotes(unittest.TestCase):
+ def test_a_note_is_printed_even_when_an_object_has_a_problem(self):
+ from codesys_synth import graphical_payload, network, vi
+ bad = b"\x0f\xdf\x02" + vi(9999) # element whose name index is not in the string table
+ with tempfile.TemporaryDirectory() as tmp:
+ paths = [os.path.join(tmp, "n%d.project" % n) for n in (1, 2)]
+ for path in paths:
+ s = Synthetic()
+ s.add("G", guid(0), lambda g, s=s: make_object(graphical_payload(
+ s.t, g, network(s.t, "c", bad), "PROGRAM G\nVAR\nEND_VAR")))
+ s.add("NoMeta", guid(0), lambda g, s=s: make_object(st_payload(s.t, g, "PROGRAM N", "x;")), meta=False)
+ s.write(path)
+ code, _, err = run_cli("diff", *paths)
+ self.assertEqual(code, codesys_project.EXIT_PROBLEMS)
+ self.assertIn("NOTE ", err)
+ self.assertIn("could not be decoded", err)
+ self.assertRegex(err, r"PROBLEM OLD .*no \.meta entry")
+ self.assertRegex(err, r"PROBLEM NEW .*no \.meta entry")
+
+ def test_a_problem_only_in_one_version_is_reported_with_its_label(self):
+ with tempfile.TemporaryDirectory() as tmp:
+ paths = []
+ for n, cut in ((1, False), (2, True)):
+ s = Synthetic()
+ t = s.t
+ obj = make_object(st_payload(t, guid(1), "PROGRAM P", "x;"))
+ s.add("P", guid(0), raw=obj[:-3] if cut else obj)
+ paths.append(os.path.join(tmp, "v%d.project" % n))
+ s.write(paths[-1])
+ code, _, err = run_cli("diff", *paths)
+ code_rev, _, err_rev = run_cli("diff", *reversed(paths))
+ self.assertEqual((code, code_rev), (codesys_project.EXIT_PROBLEMS,) * 2)
+ self.assertIn("PROBLEM NEW ", err)
+ self.assertNotIn("PROBLEM OLD ", err)
+ self.assertIn("PROBLEM OLD ", err_rev)
+ self.assertNotIn("PROBLEM NEW ", err_rev)
+
+
+class StForDeclarations(unittest.TestCase):
+ def test_a_gvl_declaration_prints_and_has_no_implementation(self):
+ with tempfile.TemporaryDirectory() as tmp:
+ path = os.path.join(tmp, "g.project")
+ s = Synthetic()
+ t = s.t
+ s.add("Vars", guid(0), lambda g: make_object(decl_only_payload(t, g, "VAR_GLOBAL\n\tg : INT;\nEND_VAR")))
+ s.write(path)
+ code, out, _ = run_cli("st", path, "Vars")
+ self.assertEqual(code, 0)
+ self.assertIn("g : INT;", out)
+ self.assertNotIn("(* implementation", out)
+ self.assertEqual(run_cli("st", path, "Vars", "--decl")[0], 0)
+ code, _, err = run_cli("st", path, "Vars", "--impl")
+ self.assertEqual(code, codesys_project.EXIT_FATAL)
+ self.assertIn("declaration only", err)
+
+
+class ListJson(unittest.TestCase):
+ def test_every_record_carries_the_full_key_set(self):
+ rec = project.ObjectRecord(guid=guid(1))
+ self.assertEqual(set(codesys_project.record_dict(rec)), {
+ "guid", "name", "kind", "subtype", "language", "path", "device", "application", "parent_guid", "problems"})
+
+
+class ConsoleEncoding(unittest.TestCase):
+ def test_a_non_ascii_comment_prints_on_a_cp1252_console(self):
+ with tempfile.TemporaryDirectory() as tmp:
+ path = os.path.join(tmp, "e.project")
+ s = Synthetic()
+ t = s.t
+ s.add("P", guid(0), lambda g: make_object(st_payload(t, g, "PROGRAM P // đ č 中", "x;")))
+ s.write(path)
+ env = dict(os.environ, PYTHONIOENCODING="cp1252", PYTHONDONTWRITEBYTECODE="1")
+ proc = subprocess.run([sys.executable, str(HERE.parent / "scripts" / "codesys_project.py"), "st", path, "P"],
+ capture_output=True, env=env)
+ self.assertEqual(proc.returncode, 0, proc.stderr)
+ self.assertIn(b"\\u4e2d", proc.stdout)
+
+
+class Survivors(unittest.TestCase):
+ """One test per guard a mutant showed no test pinned."""
+
+ def test_a_varint_of_ten_bytes_reads_and_one_of_eleven_does_not(self):
+ from codesysproj.binary import read_varint
+ self.assertEqual(read_varint(b"\x80" * 9 + b"\x01", 0), (1 << 63, 10))
+ with self.assertRaises(project.FormatError):
+ read_varint(b"\x80" * 10 + b"\x01", 0)
+
+ def test_two_slots_with_the_same_text_are_not_a_problem_and_different_text_is(self):
+ from codesys_synth import Table, slot
+ for second, expected in (("PROGRAM P", 0), ("PROGRAM Q", 1)):
+ t = Table()
+ g = guid(1)
+ data = make_object(slot(t, g, "PROGRAM P", False) + slot(t, g, second, False) + slot(t, g, "x;", True))
+ rec = project.ObjectRecord(guid=g)
+ project.classify_object(rec, data, dict(enumerate(t.texts)))
+ self.assertEqual((rec.declaration, len(rec.problems)), ("PROGRAM P", expected), second)
+
+ def test_every_var_section_alone_is_a_gvl(self):
+ from codesys_synth import Table
+ for keyword in ("VAR_GLOBAL", "VAR_CONFIG", "VAR"):
+ t = Table()
+ rec = project.ObjectRecord(guid=guid(1))
+ project.classify_object(rec, make_object(decl_only_payload(t, rec.guid, keyword + "\n\tv : INT;\nEND_VAR")),
+ dict(enumerate(t.texts)))
+ self.assertEqual((rec.kind, rec.subtype), (project.KIND_GVL, keyword))
+
+ def test_a_guid_and_a_name_resolve_in_any_case(self):
+ with tempfile.TemporaryDirectory() as tmp:
+ path = os.path.join(tmp, "c.project")
+ s = Synthetic()
+ g = s.add("Motor", guid(0), lambda g: make_object(st_payload(s.t, g, "PROGRAM Motor", "x;")))
+ s.write(path)
+ loaded = project.load_project(path)
+ for query in (g.upper(), g[:9].upper(), "MOTOR", "motor"):
+ self.assertEqual([r.guid for r in project.find_objects(loaded, query)], [g], query)
+
+ def test_a_kind_change_is_compared_and_flagged_in_both_directions(self):
+ with tempfile.TemporaryDirectory() as tmp:
+ path = os.path.join(tmp, "v.project")
+ ids = build_version(path, 1)
+ old_p = project.load_project(path)
+ ladder, st = old_p.objects[ids["ladder"]], old_p.objects[ids["edited"]]
+ for a, b in ((ladder, st), (st, ladder)):
+ result = diff.object_diff(old_p, a, old_p, b)
+ self.assertIn("graph", [sec.label for sec in result.sections])
+ text = diff.render_object_diff(result)
+ self.assertIn("# kind changed: %s -> %s" % (a.kind, b.kind), text)
+ self.assertIn(diff.LIMITS_NOTE, text)
+
+ def test_a_stable_label_keeps_the_coil_count(self):
+ from codesysproj.graph import Item
+ from codesysproj.graphview import item_label
+ self.assertEqual(item_label(Item(0, "coils", {"count": 3}), stable=True), "coils count=3")
+ self.assertEqual(item_label(Item(0, "empty", {"slot_id": 9}), stable=True), "empty")
+
+
+def _hits(decl=None, impl=None, query="x", kind=project.KIND_ST, name="Owner"):
+ rec = project.ObjectRecord(guid=guid(1), name=name, kind=kind, declaration=decl, implementation=impl)
+ proj = project.Project(path="", strings={}, objects={rec.guid: rec}, blobs={})
+ return [(h.where, h.role, h.text) for h in xref.find_references(proj, query).hits]
+
+
+class XrefNames(unittest.TestCase):
+ def test_a_non_ascii_identifier_is_scanned_not_a_crash(self):
+ impl = "caf\u00e9 := 1;\na.\u00e9 := 2;\n\u010d_start := caf\u00e9;"
+ self.assertEqual([h[:2] for h in _hits(impl=impl, query="caf\u00e9")],
+ [("implementation:1", "write"), ("implementation:3", "read")])
+ self.assertEqual([h[:2] for h in _hits(impl=impl, query="a.\u00e9")], [("implementation:2", "write")])
+ self.assertEqual(stscan.scan_paths("\u00b2 + 1"), []) # a superscript digit is a number, not a name
+
+ def test_every_name_of_a_multi_line_list_is_declared(self):
+ decl = "VAR\n\ta,\n\tb : INT;\n\tc\n\t: BOOL;\nEND_VAR"
+ for query, line in (("a", 2), ("b", 3), ("c", 4)):
+ self.assertEqual([h[:2] for h in _hits(decl, query=query, kind=project.KIND_GVL)],
+ [("declaration:%d" % line, "declared")], query)
+
+ def test_every_section_word_is_skipped(self):
+ for word in sorted(xref._SECTION_WORDS):
+ self.assertEqual([n for n, _ in xref._declared_names("%s CONSTANT n" % word.upper())], ["n"], word)
+
+ def test_extends_and_implements_are_types(self):
+ decl = "FUNCTION_BLOCK Fb EXTENDS Base IMPLEMENTS Iface, Other\nVAR\n\tm : INT;\nEND_VAR"
+ for query in ("Base", "Iface", "Other"):
+ self.assertEqual([h[:2] for h in _hits(decl, impl="", query=query)], [("declaration:1", "type")], query)
+ self.assertEqual([h[:2] for h in _hits(decl, impl="", query="m")], [("declaration:3", "declared")])
+ self.assertEqual(_hits(decl, impl="", query="extends"), [])
+ struct = "TYPE T EXTENDS U :\nSTRUCT\n\tf : INT;\nEND_STRUCT\nEND_TYPE"
+ self.assertEqual([h[:2] for h in _hits(struct, query="U", kind=project.KIND_DUT)], [("declaration:1", "type")])
+ self.assertEqual([h[:2] for h in _hits(struct, query="f", kind=project.KIND_DUT)], [("declaration:3", "declared")])
+
+ def test_an_inheritance_clause_on_its_own_line_is_types_not_a_declaration(self):
+ for keyword in ("EXTENDS", "IMPLEMENTS"):
+ decl = "FUNCTION_BLOCK P\n\t%s Base\nVAR\n\tx : INT;\nEND_VAR" % keyword
+ self.assertEqual([h[:2] for h in _hits(decl, impl="", query="x")], [("declaration:4", "declared")], keyword)
+ self.assertEqual([h[:2] for h in _hits(decl, impl="", query="Base")], [("declaration:2", "type")], keyword)
+ self.assertEqual(_hits(decl, impl="", query=keyword), [], keyword)
+
+ def test_a_pragma_line_before_the_header_leaves_the_header_intact(self):
+ decl = "{attribute 'qualified_only'}\nFUNCTION Func : BOOL\nVAR_INPUT\n\tx : INT;\nEND_VAR"
+ self.assertEqual([h[:2] for h in _hits(decl, impl="", query="Func")], [("declaration:2", "declared")])
+ self.assertEqual([h[:2] for h in _hits(decl, impl="", query="BOOL")], [("declaration:2", "type")])
+ self.assertEqual([h[:2] for h in _hits(decl, impl="", query="x")], [("declaration:4", "declared")])
+
+ def test_an_initializer_is_read_and_the_type_is_a_type(self):
+ decl = "VAR\n\tv : Lim := Lim;\nEND_VAR"
+ self.assertEqual([h[:2] for h in _hits(decl, query="Lim", kind=project.KIND_GVL)],
+ [("declaration:2", "type"), ("declaration:2", "read")])
+
+ def test_a_hit_carries_its_own_source_line(self):
+ impl = "first;\n y := x; \nlast := x"
+ self.assertEqual(_hits(impl=impl), [("implementation:2", "read", "y := x;"), ("implementation:3", "read", "last := x")])
+
+ def test_a_quoted_constant_in_a_graph_element_is_not_a_name(self):
+ from codesys_synth import element, graphical_payload, network
+ with tempfile.TemporaryDirectory() as tmp:
+ path = os.path.join(tmp, "q.project")
+ s = Synthetic()
+ t = s.t
+ items = element(t, "'abc counter'", "STRING", 0, 5111, is_input=True) + element(t, "counter", "INT", 0, 5222)
+ s.add("G", guid(0), lambda g: make_object(graphical_payload(t, g, network(t, None, items),
+ "PROGRAM G\nVAR\nEND_VAR")))
+ s.write(path)
+ hits = xref.find_references(project.load_project(path), "counter").hits
+ self.assertEqual([h.text for h in hits], ["OUT 'counter'"])
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/test_codesys_project.py b/tests/test_codesys_project.py
new file mode 100644
index 0000000..a95d634
--- /dev/null
+++ b/tests/test_codesys_project.py
@@ -0,0 +1,718 @@
+"""Reader for CODESYS 3.5 `.project` files, tested on SYNTHETIC projects only.
+
+The projects are built here from the layout the reader documents (string table, `.meta`,
+`.object`), with an encoder in codesys_synth.py and not in the reader. The format is
+undocumented and reverse-engineered, so the tests pin the reader's own rules: every guard has
+an input it must catch and a control it must accept. Names and text are invented.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import io
+import json
+import os
+import subprocess
+import sys
+import tempfile
+import unittest
+import zipfile
+from pathlib import Path
+
+HERE = Path(__file__).resolve().parent
+SCRIPTS = HERE.parent / "scripts"
+sys.path.insert(0, str(SCRIPTS))
+sys.path.insert(0, str(HERE))
+
+import codesys_project # noqa: E402
+from codesysproj import export, graph, graphview, project, zipread # noqa: E402
+from codesysproj.binary import FormatError, parse_string_table, read_varint # noqa: E402
+from codesys_synth import ( # noqa: E402
+ FILLER, Synthetic, Table, box, build_standard, coils, decl_only_payload, element, graphical_payload, guid,
+ make_meta, make_object, network, qword, slot, st_payload, vi,
+)
+
+class Base(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ cls.tmp = tempfile.TemporaryDirectory()
+ cls.path = os.path.join(cls.tmp.name, "std.project")
+ cls.ids = build_standard(cls.path)
+ cls.project = project.load_project(cls.path)
+
+ @classmethod
+ def tearDownClass(cls):
+ cls.tmp.cleanup()
+
+ def rec(self, role):
+ return self.project.objects[self.ids[role]]
+
+
+class Varints(unittest.TestCase):
+ def test_multi_byte_value(self):
+ self.assertEqual(read_varint(vi(300) + b"\xff", 0), (300, 2))
+
+ def test_truncated_varint_is_an_error_not_an_index_error(self):
+ with self.assertRaises(FormatError):
+ read_varint(b"\xac", 0)
+
+ def test_overlong_varint_is_rejected_but_ten_bytes_pass(self):
+ self.assertEqual(read_varint(vi(1 << 62), 0)[0], 1 << 62) # 9 bytes
+ with self.assertRaises(FormatError):
+ read_varint(b"\x80" * 11 + b"\x01", 0)
+
+
+class StringTable(unittest.TestCase):
+ def test_round_trip(self):
+ t = Table()
+ t.add("héllo")
+ parsed = parse_string_table(t.dump())
+ self.assertEqual(parsed[FILLER], "héllo")
+ self.assertEqual(len(parsed), FILLER + 1)
+
+ def test_entry_longer_than_file_is_an_error(self):
+ data = vi(1) + vi(0) + vi(50) + b"abc"
+ with self.assertRaisesRegex(FormatError, "runs past the end"):
+ parse_string_table(data)
+
+ def test_declared_count_must_match(self):
+ with self.assertRaisesRegex(FormatError, "declares"):
+ parse_string_table(Table().dump(declared=FILLER + 3))
+
+ def test_duplicate_index_is_an_error(self):
+ with self.assertRaisesRegex(FormatError, "twice"):
+ parse_string_table(vi(2) + vi(5) + vi(1) + b"a" + vi(5) + vi(1) + b"b")
+
+
+class EntryNames(unittest.TestCase):
+ def test_plain_and_directory_prefix(self):
+ self.assertEqual(zipread.entry_name(zipfile.ZipInfo("a.meta")), "a.meta")
+ self.assertEqual(zipread.entry_name(zipfile.ZipInfo("dir\\x.meta")), "x.meta")
+
+ def test_gbk_bytes_and_flagged_utf8(self):
+ gbk = zipfile.ZipInfo(b"\xc4\xe3.meta".decode("cp437"))
+ self.assertEqual(zipread.entry_name(gbk), "你.meta")
+ utf8 = zipfile.ZipInfo("é.meta")
+ utf8.flag_bits |= 0x800
+ self.assertEqual(zipread.entry_name(utf8), "é.meta")
+
+
+class Meta(unittest.TestCase):
+ def setUp(self):
+ self.t = Table()
+ self.own, self.parent = guid(1), guid(2)
+
+ def test_parent_and_name(self):
+ data = make_meta(self.t, self.own, self.parent, "Name")
+ self.assertEqual(project.parse_meta(data, self.own, dict(enumerate(self.t.texts))), (self.parent, "Name"))
+
+ def test_own_guid_repeated_still_gives_parent_and_name(self):
+ data = make_meta(self.t, self.own, self.parent, "Name", repeat_own=True)
+ self.assertEqual(project.parse_meta(data, self.own, dict(enumerate(self.t.texts))), (self.parent, "Name"))
+
+ def test_own_guid_missing_is_an_error(self):
+ data = make_meta(self.t, guid(9), self.parent, "Name")
+ with self.assertRaisesRegex(FormatError, "own guid not found"):
+ project.parse_meta(data, self.own, dict(enumerate(self.t.texts)))
+
+ def test_parent_must_be_guid_shaped(self):
+ data = make_meta(self.t, self.own, "not-a-guid", "Name")
+ with self.assertRaisesRegex(FormatError, "not followed by a parent guid"):
+ project.parse_meta(data, self.own, dict(enumerate(self.t.texts)))
+
+ def test_name_index_outside_the_table_is_an_error(self):
+ data = bytes([2]) + self.t.v(self.own) + self.t.v(self.parent) + vi(99999)
+ with self.assertRaisesRegex(FormatError, "not followed by a parent guid"):
+ project.parse_meta(data, self.own, dict(enumerate(self.t.texts)))
+
+ def test_stream_ending_right_after_own_guid_is_an_error(self):
+ data = bytes([2]) + self.t.v(self.own)
+ with self.assertRaisesRegex(FormatError, "not followed by a parent guid"):
+ project.parse_meta(data, self.own, dict(enumerate(self.t.texts)))
+
+
+class ObjectHeader(Base):
+ def classify(self, data, owner=None):
+ rec = project.ObjectRecord(guid=owner or guid(77))
+ project.classify_object(rec, data, self.project.strings)
+ return rec
+
+ def test_short_object_is_reported(self):
+ rec = self.classify(b"\x02\x20\x09")
+ self.assertEqual(rec.kind, "unreadable")
+ self.assertIn("shorter than", rec.problems[0])
+
+ def test_bad_magic_is_reported(self):
+ rec = self.classify(b"\xff" + make_object(b"\x01\x02")[1:])
+ self.assertEqual((rec.kind, "magic" in rec.problems[0]), ("unreadable", True))
+
+ def test_length_field_mismatch_is_reported_for_a_truncated_object(self):
+ rec = self.classify(make_object(b"\x01\x02\x03\x04")[:-2])
+ self.assertEqual(rec.kind, "unreadable")
+ self.assertIn("truncated or corrupt", rec.problems[0])
+
+ def test_well_formed_object_with_no_slots_is_other_without_problems(self):
+ rec = self.classify(make_object(b"\x0f\x01\x02\x03"))
+ self.assertEqual((rec.kind, rec.problems), ("other", []))
+
+
+class TextSlots(unittest.TestCase):
+ def setUp(self):
+ self.t = Table()
+ self.owner = guid(5)
+
+ def classify(self, payload):
+ rec = project.ObjectRecord(guid=self.owner)
+ project.classify_object(rec, make_object(payload), dict(enumerate(self.t.texts)))
+ return rec
+
+ def test_st_text_is_read_with_line_endings_intact(self):
+ rec = self.classify(st_payload(self.t, self.owner, "PROGRAM P\r\nVAR\r\nEND_VAR", "a := 1;\r\n"))
+ self.assertEqual((rec.kind, rec.declaration, rec.implementation),
+ ("st", "PROGRAM P\r\nVAR\r\nEND_VAR", "a := 1;\r\n"))
+
+ def test_marker_with_the_wrong_back_reference_is_ignored(self):
+ decoy = b"\x41\xe5\x02" + self.t.v("DECOY TEXT") + self.t.v(guid(99) + "_Decl_LineIds")
+ rec = self.classify(decoy + st_payload(self.t, self.owner, "PROGRAM P", "real;"))
+ self.assertEqual((rec.declaration, rec.implementation), ("PROGRAM P", "real;"))
+
+ def test_decl_reference_does_not_satisfy_an_impl_slot(self):
+ wrong = b"\x5b\xe5\x02" + self.t.v("x") + self.t.v(self.owner + "_Decl_LineIds")
+ rec = self.classify(wrong + decl_only_payload(self.t, self.owner, "VAR_GLOBAL"))
+ self.assertEqual((rec.kind, rec.implementation), ("gvl", None))
+
+ def test_text_index_outside_the_table_is_reported(self):
+ ref = self.t.v(self.owner + "_Decl_LineIds")
+ rec = self.classify(b"\x41\xe5\x02" + vi(99999) + ref)
+ self.assertEqual(rec.declaration, None)
+ self.assertIn("not in the string table", rec.problems[0])
+
+ def test_two_different_slots_use_the_first_and_say_so(self):
+ payload = (decl_only_payload(self.t, self.owner, "VAR_GLOBAL a")
+ + slot(self.t, self.owner, "VAR_GLOBAL b", False))
+ rec = self.classify(payload)
+ self.assertEqual(rec.declaration, "VAR_GLOBAL a")
+ self.assertIn("using the first", rec.problems[0])
+
+ def test_truncated_slot_after_the_marker_is_not_a_crash(self):
+ rec = self.classify(b"\x0f\x41\xe5\x02\x80")
+ self.assertEqual((rec.kind, rec.problems), ("other", []))
+
+
+class Kinds(Base):
+ def test_kinds_by_payload(self):
+ self.assertEqual(self.rec("DevA.pou").kind, "st")
+ self.assertEqual(self.rec("gvl").kind, "gvl")
+ self.assertEqual(self.rec("dut").kind, "dut")
+ self.assertEqual(self.rec("other").kind, "other")
+ self.assertEqual(self.rec("graph").kind, "graphical")
+
+ def test_subtype_skips_comments_and_pragmas(self):
+ self.assertEqual(self.rec("pragma").subtype, "FUNCTION_BLOCK")
+ self.assertEqual(self.rec("DevA.pou").subtype, "PROGRAM")
+
+ def test_declaration_keyword_needs_a_declaration(self):
+ self.assertIsNone(project.declaration_keyword("x := 1;"))
+ self.assertEqual(project.declaration_keyword(" FUNCTION f : INT"), "FUNCTION")
+
+ def test_graphical_language_is_read_and_a_declaration_survives(self):
+ rec = self.rec("graph")
+ self.assertEqual(rec.language, "Ld")
+ self.assertTrue(rec.declaration.startswith("PROGRAM Ladder"))
+ self.assertIsNone(rec.implementation)
+
+ def test_language_is_none_without_the_header_prefix_or_with_too_few_values(self):
+ t = Table()
+ owner = guid(5)
+ for payload in (b"\x0f\xe1\x02" + b"\x00" * 5 + b"\x27", b"\x0f\xe6\x02\x48\x0f\xe1\x02"):
+ rec = project.ObjectRecord(guid=owner)
+ project.classify_object(rec, make_object(payload), dict(enumerate(t.texts)))
+ self.assertEqual((rec.kind, rec.language), ("graphical", None))
+
+ def test_a_network_tag_other_than_e1_still_marks_graphical(self):
+ t = Table()
+ owner = guid(5)
+ rec = project.ObjectRecord(guid=owner)
+ payload = graphical_payload(t, owner, network(t, "c", tag=b"\xe7\x02"), "PROGRAM G")
+ project.classify_object(rec, make_object(payload), dict(enumerate(t.texts)))
+ self.assertEqual(rec.kind, "graphical")
+
+
+class Tree(Base):
+ def test_path_device_and_application(self):
+ rec = self.rec("DevB.pou")
+ self.assertEqual(rec.path, ["DevB", "Plc Logic", "Application"])
+ self.assertEqual((rec.device, rec.application), ("DevB", "Application"))
+
+ def test_non_guid_entries_are_not_objects(self):
+ self.assertEqual(len(self.project.objects), 13)
+
+ def test_object_outside_a_plc_logic_node_has_no_application(self):
+ rec = self.rec("other")
+ self.assertEqual((rec.path, rec.application), (["DevA"], None))
+
+ def test_records_in_tree_order(self):
+ names = [r.name for r in self.project.records() if r.kind == "st"]
+ self.assertEqual(names, ["Main", "Pragma", "Main"]) # by ancestors (DevA before DevB), then name
+
+ def test_parent_loop_is_reported_not_followed(self):
+ a, b = guid(1), guid(2)
+ objects = {a: project.ObjectRecord(a, "A", b), b: project.ObjectRecord(b, "B", a)}
+ project._link_parents(objects)
+ self.assertIn("loops back", objects[a].problems[0])
+ self.assertEqual(objects[a].path, ["B"])
+
+ def test_linear_chain_has_no_loop_problem(self):
+ a, b = guid(1), guid(2)
+ objects = {a: project.ObjectRecord(a, "A", guid(0)), b: project.ObjectRecord(b, "B", a)}
+ project._link_parents(objects)
+ self.assertEqual((objects[b].path, objects[b].problems, objects[a].problems), (["A"], [], []))
+
+
+class Damaged(unittest.TestCase):
+ def test_damaged_entries_are_reported_and_the_rest_still_read(self):
+ s = Synthetic()
+ t = s.t
+ null = guid(0)
+ s.add("Good", null, lambda g: make_object(st_payload(t, g, "PROGRAM Good", "x;")))
+ s.add("Cut", null, lambda g: make_object(st_payload(t, g, "PROGRAM Cut", "x;"))[:-4])
+ s.add("Orphan", null, meta=False, payload=make_object(b"\x01"))
+ s.add("NoObject", null)
+ with tempfile.TemporaryDirectory() as tmp:
+ path = os.path.join(tmp, "d.project")
+ s.write(path)
+ with zipfile.ZipFile(path, "a") as z: # a meta whose stream never names its own guid
+ z.writestr(guid(99) + ".meta", bytes([2, 0x20]))
+ proj = project.load_project(path)
+ by_name = {r.name: r for r in proj.objects.values()}
+ self.assertEqual(by_name["Good"].kind, "st")
+ self.assertEqual(by_name["Good"].problems, [])
+ self.assertEqual(by_name["Cut"].kind, "unreadable")
+ orphan = proj.objects[guid(3)]
+ self.assertEqual((orphan.name, orphan.has_meta), (None, False))
+ self.assertIn("no .meta", orphan.problems[0])
+ self.assertIn("no .object", by_name["NoObject"].problems[0])
+ self.assertIn("meta:", proj.objects[guid(99)].problems[0])
+
+ def test_not_a_zip_and_no_string_table_are_fatal(self):
+ with tempfile.TemporaryDirectory() as tmp:
+ bad = os.path.join(tmp, "x.project")
+ Path(bad).write_bytes(b"not a zip")
+ with self.assertRaisesRegex(FormatError, "not a readable zip"):
+ project.load_project(bad)
+ with self.assertRaisesRegex(FormatError, "not a readable zip"):
+ project.load_project(os.path.join(tmp, "missing.project"))
+ empty = os.path.join(tmp, "e.project")
+ with zipfile.ZipFile(empty, "w") as z:
+ z.writestr("readme.txt", b"x")
+ with self.assertRaisesRegex(FormatError, "no __shared_data_storage_string_table__"):
+ project.load_project(empty)
+
+ def test_corrupt_string_table_is_fatal(self):
+ s = Synthetic()
+ s.add("A", guid(0))
+ with tempfile.TemporaryDirectory() as tmp:
+ path = os.path.join(tmp, "c.project")
+ s.write(path, declared=FILLER + 50)
+ with self.assertRaisesRegex(FormatError, "declares"):
+ project.load_project(path)
+
+
+class Lookup(Base):
+ def test_full_guid_prefix_and_name(self):
+ g = self.ids["gvl"]
+ self.assertEqual([r.guid for r in project.find_objects(self.project, g)], [g])
+ self.assertEqual([r.guid for r in project.find_objects(self.project, g[:8])], [g])
+ self.assertEqual([r.guid for r in project.find_objects(self.project, "Globals")], [g])
+
+ def test_short_prefix_is_not_a_guid(self):
+ self.assertEqual(project.find_objects(self.project, self.ids["gvl"][:7]), [])
+
+ def test_same_name_in_two_devices_returns_both(self):
+ self.assertEqual(len(project.find_objects(self.project, "Main")), 2)
+
+ def test_ambiguous_name_is_an_error_that_lists_the_candidates(self):
+ with self.assertRaises(FormatError) as ctx:
+ codesys_project.resolve_one(self.project, "Main")
+ message = str(ctx.exception)
+ self.assertIn("ambiguous", message)
+ self.assertIn("DevA/Plc Logic/Application/Main", message)
+ self.assertIn("DevB/Plc Logic/Application/Main", message)
+
+ def test_unknown_name_is_an_error(self):
+ with self.assertRaisesRegex(FormatError, "no object"):
+ codesys_project.resolve_one(self.project, "Nope")
+
+
+class FileNames(unittest.TestCase):
+ def test_separators_and_leading_dots_are_neutralised(self):
+ self.assertEqual(export.safe_filename_part("a/b\\c"), "a_b_c")
+ self.assertEqual(export.safe_filename_part("../x"), "_x")
+ self.assertEqual(export.safe_filename_part(".."), "_")
+ self.assertEqual(export.safe_filename_part(""), "_")
+
+ def test_ordinary_names_survive(self):
+ self.assertEqual(export.safe_filename_part("Pumpa_1 ćš.v2-x"), "Pumpa_1 ćš.v2-x")
+
+ def test_long_names_are_cut(self):
+ self.assertEqual(len(export.safe_filename_part("n" * 500)), export.MAX_STEM_PART)
+
+
+class DumpSt(Base):
+ def test_st_pous_and_declarations_are_written_and_collisions_get_the_guid(self):
+ with tempfile.TemporaryDirectory() as tmp:
+ out = os.path.join(tmp, "out")
+ written = export.dump_st(self.project, out)
+ names = sorted(os.listdir(out))
+ self.assertEqual(len(written), 5)
+ self.assertEqual(names, sorted([
+ "DevA_Pragma.decl.st", "DevA_Pragma.impl.st",
+ "DevA_Main.decl.st", "DevA_Main.impl.st", "DevB_Main.decl.st", "DevB_Main.impl.st",
+ "DevA_Globals.decl.st", "DevA_Point.decl.st"]))
+ # Globals (gvl) and Point (dut) have a declaration only; Ladder (graphical) and Tasks (other) are not written
+
+ def test_text_is_byte_exact(self):
+ with tempfile.TemporaryDirectory() as tmp:
+ export.dump_st(self.project, tmp)
+ raw = Path(tmp, "DevB_Main.impl.st").read_bytes()
+ self.assertEqual(raw, b"x := x + 1; // DevB\r\n")
+
+ def test_same_device_same_name_gets_a_guid_suffix_and_nothing_is_overwritten(self):
+ s = Synthetic()
+ t = s.t
+ d = s.add("Dev", guid(0))
+ first = s.add("Dup", d, lambda g: make_object(st_payload(t, g, "PROGRAM Dup", "one;")))
+ second = s.add("dup", d, lambda g: make_object(st_payload(t, g, "PROGRAM dup", "two;")))
+ with tempfile.TemporaryDirectory() as tmp:
+ path = os.path.join(tmp, "p.project")
+ s.write(path)
+ out = os.path.join(tmp, "o")
+ export.dump_st(project.load_project(path), out)
+ names = sorted(os.listdir(out))
+ texts = sorted(Path(out, n).read_text() for n in names if n.endswith(".impl.st"))
+ self.assertEqual(len(names), 4)
+ self.assertTrue(all(g[:8] in n for g in (first, second) for n in names if g[:8] in n))
+ self.assertTrue(all("_" + first[:8] in n or "_" + second[:8] in n for n in names))
+ self.assertEqual(texts, ["one;", "two;"])
+
+ def test_object_without_meta_and_object_without_ancestors_get_stable_names(self):
+ s = Synthetic()
+ t = s.t
+ s.add("RootPou", guid(0), lambda g: make_object(st_payload(t, g, "PROGRAM R", "r;")))
+ lost = s.add("x", guid(0), lambda g: make_object(st_payload(t, g, "PROGRAM L", "l;")), meta=False)
+ with tempfile.TemporaryDirectory() as tmp:
+ path = os.path.join(tmp, "p.project")
+ s.write(path)
+ out = os.path.join(tmp, "o")
+ export.dump_st(project.load_project(path), out)
+ names = sorted(os.listdir(out))
+ self.assertEqual(names, sorted(["root_RootPou.decl.st", "root_RootPou.impl.st",
+ "unnamed_%s.decl.st" % lost, "unnamed_%s.impl.st" % lost]))
+
+ def test_hostile_names_stay_inside_the_output_directory(self):
+ s = Synthetic()
+ t = s.t
+ d = s.add("../../dev", guid(0))
+ s.add("../evil/x", d, lambda g: make_object(st_payload(t, g, "PROGRAM E", "e;")))
+ with tempfile.TemporaryDirectory() as tmp:
+ path = os.path.join(tmp, "p.project")
+ s.write(path)
+ out = os.path.join(tmp, "a", "b")
+ export.dump_st(project.load_project(path), out)
+ inside = os.listdir(out)
+ outside = [n for root, _, files in os.walk(tmp) for n in files if os.path.dirname(os.path.join(root, n)) != out
+ and not n.endswith(".project")]
+ self.assertEqual(len(inside), 2)
+ self.assertEqual(outside, [])
+
+
+class GraphDecode(Base):
+ def decode(self):
+ rec = self.rec("graph")
+ return graph.decode_graph(rec, self.project.blobs[rec.guid], self.project.strings)
+
+ def test_networks_comments_and_tags(self):
+ result = self.decode()
+ self.assertEqual([n.comment for n in result.networks], ["start the lamp", None])
+ self.assertEqual([n.tag for n in result.networks], ["e1 02", "e7 02"])
+ self.assertEqual(result.language, "Ld")
+
+ def test_elements_carry_decoded_fields_and_heuristic_labels(self):
+ items = {i.decoded.get("name"): i for i in self.decode().networks[0].items if i.kind == "element"}
+ lamp, start, preset = items["Lamp"], items["Start"], items["Preset"]
+ self.assertEqual((lamp.decoded["type"], lamp.decoded["modifier"]), ("BOOL", 2))
+ self.assertEqual((lamp.heuristic["direction"], lamp.heuristic["modifier_meaning"], lamp.heuristic["role"]),
+ ("OUT", "set", "coil"))
+ self.assertEqual((start.heuristic["direction"], start.heuristic["modifier_meaning"]), ("IN", "negated"))
+ self.assertNotIn("role", start.heuristic)
+ self.assertEqual(preset.heuristic["modifier_meaning"], "none")
+ self.assertEqual((lamp.ids, start.ids, preset.ids), ([5111], [5230], [5340, 6035])) # the 733 filler id is dropped
+
+ def test_a_coil_role_stops_after_the_declared_count(self):
+ t = Table()
+ items = coils(1) + element(t, "A", "BOOL", 0, 5111) + element(t, "B", "BOOL", 0, 5112)
+ rec = project.ObjectRecord(guid=guid(5))
+ data = make_object(graphical_payload(t, rec.guid, network(t, "", items), "PROGRAM G"))
+ roles = [i.heuristic.get("role") for i in graph.decode_graph(rec, data, dict(enumerate(t.texts))).networks[0].items
+ if i.kind == "element"]
+ self.assertEqual(roles, ["coil", None])
+
+ def test_coil_role_goes_only_to_outputs(self):
+ result = self.decode_items(lambda t: coils(2) + element(t, "In", "BOOL", 0, 5111, is_input=True)
+ + element(t, "Out", "BOOL", 0, 5112))
+ roles = {i.decoded["name"]: i.heuristic.get("role") for i in result.networks[0].items if i.kind == "element"}
+ self.assertEqual(roles, {"In": None, "Out": "coil"})
+
+ def test_box_decodes_type_instance_header_id_and_nesting(self):
+ t = Table()
+ inner = element(t, "In1", "INT", 0, 5560, is_input=True)
+ items = box(t, "ADD", "", 6034) + box(t, "TON", "Timer1", 6140) + inner + qword(6141) + qword(6035)
+ rec = project.ObjectRecord(guid=guid(5))
+ data = make_object(graphical_payload(t, rec.guid, network(t, "", items), "PROGRAM G"))
+ net = graph.decode_graph(rec, data, dict(enumerate(t.texts))).networks[0]
+ boxes = [i for i in net.items if i.kind == "box"]
+ self.assertEqual([(b.decoded["box_type"], b.decoded["instance"], b.decoded["header_id"]) for b in boxes],
+ [("ADD", "", 6034), ("TON", "Timer1", 6140)])
+ depth = {i.decoded.get("name") or i.decoded.get("box_type"): i.heuristic["depth"] for i in net.items
+ if i.kind in ("box", "element")}
+ self.assertEqual(depth, {"ADD": 0, "TON": 1, "In1": 2})
+
+ def test_box_header_id_sentinel_means_not_a_box(self):
+ t = Table()
+ items = box(t, "ADD", "", graph.SENTINEL_ID)
+ rec = project.ObjectRecord(guid=guid(5))
+ data = make_object(graphical_payload(t, rec.guid, network(t, "", items), "PROGRAM G"))
+ self.assertEqual([i.kind for i in graph.decode_graph(rec, data, dict(enumerate(t.texts))).networks[0].items], [])
+
+ def test_pins_and_empty_slots(self):
+ kinds = [i.kind for i in self.decode().networks[0].items]
+ self.assertIn("pins", kinds)
+ empty = [i for i in self.decode().networks[0].items if i.kind == "empty"][0]
+ self.assertEqual((empty.decoded["id"], empty.decoded["raw"]), (7451, "42 00 00 00 00"))
+ pin_item = [i for i in self.decode().networks[0].items if i.kind == "pins"][0]
+ self.assertEqual(pin_item.decoded["names"], ["IN", "PT"])
+
+ def decode_items(self, items, comment=""):
+ t = self.table = Table()
+ rec = project.ObjectRecord(guid=guid(5))
+ data = make_object(graphical_payload(t, rec.guid, network(t, comment, items(t) if callable(items) else items),
+ "PROGRAM G"))
+ return graph.decode_graph(rec, data, dict(enumerate(t.texts)))
+
+ def decode_at_end(self, raw):
+ """Records that run into the end of the object (no declaration slot after them)."""
+ t = Table()
+ rec = project.ObjectRecord(guid=guid(5))
+ data = make_object(network(t, "") + raw)
+ return graph.decode_graph(rec, data, dict(enumerate(t.texts))).networks[0].items
+
+ def test_logic_box_tag_and_a_box_cannot_be_made_from_a_bare_type(self):
+ result = self.decode_items(lambda t: box(t, "AND", "", 6034, tag=b"\xe2\x02"))
+ self.assertEqual([(i.kind, i.decoded["tag"]) for i in result.networks[0].items], [("box", "e2 02")])
+ bare = self.decode_items(lambda t: box(t, "AND", "", 6034).replace(b"\xde\x02\x00", b"\x11\x22\x33"))
+ self.assertEqual(bare.networks[0].items, []) # no instance marker after the type: not a box
+
+ def test_box_header_needs_the_01_flag_and_an_id_within_range(self):
+ flagless = lambda t: box(t, "AND", "", 6034).replace(b"\x01" + qword(6034), b"\x02" + qword(6034))
+ self.assertEqual(self.decode_items(flagless).networks[0].items, [])
+ self.assertEqual(self.decode_items(lambda t: box(t, "AND", "", graph.MAX_ID + 1)).networks[0].items, [])
+ self.assertEqual(len(self.decode_items(lambda t: box(t, "AND", "", graph.MAX_ID)).networks[0].items), 1)
+
+ def test_box_with_an_unresolved_type_is_not_a_box(self):
+ raw = b"\x0f\xe0\x02" + vi(99999) + b"\xde\x02\x00\x27\x27\x27\x27\xdd\x02" + b"\x00" * 8 + b"\x01" + qword(6034)
+ self.assertEqual(self.decode_items(raw).networks[0].items, [])
+
+ def test_span_end_exists_only_when_the_closing_id_is_found(self):
+ lone = self.decode_items(lambda t: box(t, "AND", "", 6034)).networks[0].items[0]
+ closed = self.decode_items(lambda t: box(t, "AND", "", 6034) + qword(6035)).networks[0].items[0]
+ self.assertNotIn("span_end", lone.heuristic)
+ self.assertIn("span_end", closed.heuristic)
+
+ def test_element_with_empty_name_or_empty_type_or_no_modifier(self):
+ nameless = self.decode_items(b"\xdf\x02\x27\x27\x27\x27\xdd\x02\x00" + b"\x00" * 8).networks[0].items[0]
+ self.assertEqual((nameless.decoded["name"], "type" in nameless.decoded), ("", False))
+ typeless = self.decode_items(lambda t: b"\xdf\x02" + t.v("V") + b"\x27\x27\x27\xdd\x02\x00" + b"\x00" * 8)
+ self.assertEqual((typeless.networks[0].items[0].decoded["name"], "type" in typeless.networks[0].items[0].decoded),
+ ("V", False))
+ no_mod = self.decode_items(lambda t: b"\xdf\x02" + t.v("V") + t.v("INT") + b"\x01" * 60).networks[0].items[0]
+ self.assertIsNone(no_mod.decoded["modifier"])
+ self.assertNotIn("modifier_meaning", no_mod.heuristic)
+
+ def test_unlisted_modifier_code_is_not_given_a_meaning(self):
+ item = self.decode_items(lambda t: element(t, "A", "BOOL", 16, 5111)).networks[0].items[0]
+ self.assertEqual((item.decoded["modifier"], item.heuristic["modifier_meaning"]), (16, "unknown code"))
+
+ def test_markers_pins_and_empty_need_their_exact_shape(self):
+ res = self.decode_items(b"\x0f\xa6\x01\x02").networks[0].items
+ self.assertEqual([(i.kind, i.decoded["value"]) for i in res], [("parallel_marker", 2)])
+ self.assertEqual(self.decode_at_end(b"\x0f\x6f\x6c"), []) # cut off by the end of the object
+ self.assertEqual(self.decode_at_end(b"\x0f\xa6\x01"), [])
+ self.assertEqual([i.kind for i in self.decode_at_end(b"\x0f\x6f\x6c\x01")], ["coils"])
+ self.assertEqual(self.decode_items(lambda t: bytes([0x6A, 2, 0x0F]) + t.v("A") + t.v("B")).networks[0].items, [])
+ self.assertEqual(self.decode_at_end(b"\x6a\x02\x0e\x80"), []) # truncated pin name
+ self.assertEqual(self.decode_items(b"\x9a\x01\x42\x00\x00\x00\x00\x01\x02\x03").networks[0].items, [])
+
+ def test_networks_come_out_in_file_order_whatever_their_tag(self):
+ t = Table()
+ rec = project.ObjectRecord(guid=guid(5))
+ nets = network(t, "first", tag=b"\xe7\x02") + network(t, "second") + network(t, "third", tag=b"\xe7\x02")
+ data = make_object(graphical_payload(t, rec.guid, nets, "PROGRAM G"))
+ result = graph.decode_graph(rec, data, dict(enumerate(t.texts)))
+ self.assertEqual([n.comment for n in result.networks], ["first", "second", "third"])
+
+ def test_record_ending_inside_the_comment_is_noted(self):
+ t = Table()
+ rec = project.ObjectRecord(guid=guid(5))
+ result = graph.decode_graph(rec, make_object(b"\x0f\xe1\x02" + b"\x00" * 5 + b"\x80"), dict(enumerate(t.texts)))
+ self.assertIn("ends inside the comment", result.notes[0])
+
+ def test_unresolved_element_name_is_an_unknown_record_not_a_guess(self):
+ t = Table()
+ broken = b"\xdf\x02" + vi(88888) + b"\x27\x27\x27\xdd\x02\x00" + b"\x00" * 8
+ items = broken + element(t, "Fine", "INT", 0, 5111)
+ rec = project.ObjectRecord(guid=guid(5))
+ data = make_object(graphical_payload(t, rec.guid, network(t, "", items), "PROGRAM G"))
+ result = graph.decode_graph(rec, data, dict(enumerate(t.texts)))
+ kinds = [(i.kind, i.decoded.get("name")) for i in result.networks[0].items]
+ self.assertEqual(kinds, [("unknown", None), ("element", "Fine")])
+ self.assertIn("could not be decoded", result.notes[0])
+ self.assertIn("88888", result.networks[0].items[0].decoded["reason"])
+
+ def test_element_cut_off_by_the_end_of_the_object_is_unknown(self):
+ t = Table()
+ rec = project.ObjectRecord(guid=guid(5))
+ data = make_object(network(t, "") + b"\xdf\x02\x80")
+ result = graph.decode_graph(rec, data, dict(enumerate(t.texts)))
+ self.assertEqual([i.kind for i in result.networks[0].items], ["unknown"])
+
+ def test_unresolved_comment_index_is_noted(self):
+ t = Table()
+ rec = project.ObjectRecord(guid=guid(5))
+ data = make_object(b"\x0f\xe1\x02" + b"\x00" * 5 + vi(77777) + b"\x27\x27")
+ result = graph.decode_graph(rec, data, dict(enumerate(t.texts)))
+ self.assertIn("comment index 77777", result.notes[0])
+
+ def test_object_without_networks_says_so(self):
+ rec = self.rec("DevA.pou")
+ result = graph.decode_graph(rec, self.project.blobs[rec.guid], self.project.strings)
+ self.assertEqual(result.networks, [])
+ self.assertIn("no network records", result.notes[0])
+
+ def test_text_and_json_labels(self):
+ result = self.decode()
+ text = graphview.render_graph_text(result)
+ self.assertIn("NOT decoded: wiring", text)
+ self.assertIn("not full coverage", text)
+ self.assertIn("OUT/coil 'Lamp' type=BOOL mod=2 ~set", text)
+ self.assertIn("IN 'Start' type=BOOL mod=1 ~negated", text)
+ self.assertIn("comment='start the lamp'", text)
+ data = graphview.graph_to_dict(result)
+ item = data["networks"][0]["items"][1]
+ self.assertIn("ids", item["heuristic"])
+ self.assertNotIn("ids", item["decoded"])
+ self.assertIn("wiring", data["legend"])
+ self.assertIn("not full coverage", data["legend"])
+
+
+class Cli(Base):
+ def run_cli(self, *argv):
+ out, err = io.StringIO(), io.StringIO()
+ with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err):
+ code = codesys_project.main(list(argv))
+ return code, out.getvalue(), err.getvalue()
+
+ def test_list_json_has_every_object_with_kind_and_chain(self):
+ code, out, _ = self.run_cli("list", self.path, "--json")
+ rows = json.loads(out)
+ self.assertEqual(code, 0)
+ self.assertEqual(len(rows), len(self.project.objects))
+ ladder = [r for r in rows if r["name"] == "Ladder"][0]
+ self.assertEqual((ladder["kind"], ladder["language"], ladder["path"]), ("graphical", "Ld", ["DevA", "Plc Logic", "Application"]))
+
+ def test_list_kind_filter(self):
+ code, out, _ = self.run_cli("list", self.path, "--kind", "gvl")
+ lines = [line for line in out.splitlines() if not line.startswith("#")]
+ self.assertEqual((code, len(lines)), (0, 1))
+ self.assertIn("Globals", lines[0])
+
+ def test_st_prints_declaration_and_implementation(self):
+ code, out, _ = self.run_cli("st", self.path, self.ids["DevA.pou"][:8])
+ self.assertEqual(code, 0)
+ self.assertIn("PROGRAM Main", out)
+ self.assertIn("x := x + 1; // DevA", out)
+
+ def test_st_decl_and_impl_flags_are_raw(self):
+ _, decl, _ = self.run_cli("st", self.path, self.ids["DevA.pou"], "--decl")
+ _, impl, _ = self.run_cli("st", self.path, self.ids["DevA.pou"], "--impl")
+ self.assertTrue(decl.startswith("PROGRAM Main") and "x :=" not in decl)
+ self.assertEqual(impl, "x := x + 1; // DevA\r\n")
+
+ def test_st_of_a_graphical_pou_prints_the_declaration_only(self):
+ code, out, _ = self.run_cli("st", self.path, "Ladder")
+ self.assertEqual(code, 0)
+ self.assertIn("PROGRAM Ladder", out)
+ self.assertIn("graphical (Ld), not ST", out)
+ code, _, err = self.run_cli("st", self.path, "Ladder", "--impl")
+ self.assertEqual(code, 2)
+ self.assertIn("graphical", err)
+
+ def test_st_refuses_non_st_and_ambiguous_names(self):
+ code, _, err = self.run_cli("st", self.path, "Tasks")
+ self.assertEqual((code, "no ST text" in err), (2, True))
+ code, _, err = self.run_cli("st", self.path, "Main")
+ self.assertEqual((code, "ambiguous" in err), (2, True))
+
+ def test_graph_refuses_non_graphical(self):
+ code, _, err = self.run_cli("graph", self.path, "Globals")
+ self.assertEqual((code, "not graphical" in err), (2, True))
+
+ def test_graph_json_and_text(self):
+ code, out, _ = self.run_cli("graph", self.path, "Ladder", "--json")
+ self.assertEqual((code, json.loads(out)["networks"][0]["comment"]), (0, "start the lamp"))
+ code, out, _ = self.run_cli("graph", self.path, "Ladder")
+ self.assertEqual(code, 0)
+ self.assertIn("network 2", out)
+
+ def test_graph_exits_1_when_something_could_not_be_decoded(self):
+ s = Synthetic()
+ t = s.t
+ s.add("Bad", guid(0), lambda g: make_object(graphical_payload(t, g, network(t, "", b"\xdf\x02" + vi(88888)), "PROGRAM Bad")))
+ with tempfile.TemporaryDirectory() as tmp:
+ path = os.path.join(tmp, "b.project")
+ s.write(path)
+ code, out, _ = self.run_cli("graph", path, "Bad")
+ self.assertEqual(code, 1)
+ self.assertIn("UNKNOWN", out)
+
+ def test_dump_st_command(self):
+ with tempfile.TemporaryDirectory() as tmp:
+ code, out, _ = self.run_cli("dump-st", self.path, tmp)
+ self.assertEqual((code, len(os.listdir(tmp))), (0, 8))
+ self.assertIn("wrote 5 object(s)", out)
+
+ def test_exit_codes_for_problems_and_unreadable_input(self):
+ s = Synthetic()
+ t = s.t
+ s.add("Good", guid(0), lambda g: make_object(st_payload(t, g, "PROGRAM Good", "x;")))
+ s.add("Cut", guid(0), lambda g: make_object(st_payload(t, g, "PROGRAM Cut", "x;"))[:-3])
+ with tempfile.TemporaryDirectory() as tmp:
+ path = os.path.join(tmp, "c.project")
+ s.write(path)
+ code, out, err = self.run_cli("list", path)
+ self.assertEqual(code, 1)
+ self.assertIn("Good", out)
+ self.assertIn("PROBLEM", err)
+ garbage = os.path.join(tmp, "g.project")
+ Path(garbage).write_bytes(b"nope")
+ code, _, err = self.run_cli("list", garbage)
+ self.assertEqual((code, "not a readable zip" in err), (2, True))
+
+ def test_script_runs_as_a_program(self):
+ done = subprocess.run([sys.executable, "-B", str(SCRIPTS / "codesys_project.py"), "list", self.path, "--kind", "st"],
+ capture_output=True, text=True)
+ self.assertEqual(done.returncode, 0)
+ self.assertEqual(len([ln for ln in done.stdout.splitlines() if not ln.startswith("#")]), 3)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/test_codesys_tools.py b/tests/test_codesys_tools.py
new file mode 100644
index 0000000..4acc207
--- /dev/null
+++ b/tests/test_codesys_tools.py
@@ -0,0 +1,700 @@
+"""`info`, `diff` and `xref` of the CODESYS `.project` reader, tested on SYNTHETIC projects only.
+
+Two versions of one project are built in codesys_synth.py (`build_version`) with known changes,
+and a project made for xref (`build_xref`). Every guard has an input it must catch and a control
+it must accept. Names and text are invented.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import io
+import json
+import os
+import sys
+import tempfile
+import unittest
+import zipfile
+from pathlib import Path
+
+HERE = Path(__file__).resolve().parent
+sys.path.insert(0, str(HERE.parent / "scripts"))
+sys.path.insert(0, str(HERE))
+
+import codesys_project # noqa: E402
+from codesysproj import buildinfo, diff, project, stscan, xref # noqa: E402
+from codesysproj.binary import FormatError # noqa: E402
+from codesys_synth import ( # noqa: E402
+ Synthetic, slot, build_version, build_xref, decl_only_payload, devices_meta_xml, guid, make_object, st_payload,
+ tools_versions_xml,
+)
+
+
+def run_cli(*argv):
+ out, err = io.StringIO(), io.StringIO()
+ with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err):
+ code = codesys_project.main(list(argv))
+ return code, out.getvalue(), err.getvalue()
+
+
+def write_entries(path, entries):
+ """A project holding only the given extra entries (no objects) plus an empty string table."""
+ s = Synthetic()
+ s.write(path, extra=entries)
+
+
+class Versions(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ cls.tmp = tempfile.TemporaryDirectory()
+ cls.old = os.path.join(cls.tmp.name, "v1.project")
+ cls.new = os.path.join(cls.tmp.name, "v2.project")
+ cls.ids = build_version(cls.old, 1)
+ cls.ids2 = build_version(cls.new, 2)
+
+ @classmethod
+ def tearDownClass(cls):
+ cls.tmp.cleanup()
+
+
+class Info(unittest.TestCase):
+ def setUp(self):
+ self.tmp = tempfile.TemporaryDirectory()
+ self.addCleanup(self.tmp.cleanup)
+ self.path = os.path.join(self.tmp.name, "p.project")
+
+ def test_reads_every_device_and_application(self):
+ write_entries(self.path, {"devicesmetainformation.auxiliary": devices_meta_xml([
+ ("DevA", [("App1", "11223344-0000-0000-0000-000000000000", "55667788-0000-0000-0000-000000000000"),
+ ("App2", "99aabbcc-0000-0000-0000-000000000000", "ddeeff00-0000-0000-0000-000000000000")]),
+ ("DevB", [("App1", "01020304-0000-0000-0000-000000000000", "05060708-0000-0000-0000-000000000000")]),
+ ])})
+ info = buildinfo.read_info(self.path)
+ self.assertEqual([d.name for d in info.devices], ["DevA", "DevB"])
+ a = info.devices[0]
+ self.assertEqual((a.device_type, a.device_id, a.device_version, a.device_name),
+ ("4096", "0000 0001", "1.2.3.4", "Model-DevA"))
+ self.assertEqual([x.name for x in a.applications], ["App1", "App2"])
+ self.assertEqual(a.applications[1].code_id, "99aabbcc-0000-0000-0000-000000000000")
+ self.assertEqual(a.applications[1].data_id, "ddeeff00-0000-0000-0000-000000000000")
+ self.assertEqual(len(info.devices[1].applications), 1)
+ self.assertEqual(info.notes, [])
+
+ def test_short_and_byte_swapped_forms(self):
+ write_entries(self.path, {"devicesmetainformation.auxiliary": devices_meta_xml([
+ ("DevA", [("App", "11223344-0000-0000-0000-000000000000", "55667788-0000-0000-0000-000000000000")])])})
+ app = buildinfo.read_info(self.path).devices[0].applications[0]
+ self.assertEqual(buildinfo.short_id(app.code_id), "11223344")
+ self.assertEqual(buildinfo.swapped_id(app.code_id), "44332211")
+
+ def test_short_and_swapped_refuse_what_is_not_a_guid(self):
+ self.assertIsNone(buildinfo.short_id("not a guid"))
+ self.assertIsNone(buildinfo.swapped_id("not a guid"))
+ self.assertIsNone(buildinfo.short_id(None))
+
+ def test_text_names_both_forms_and_the_limits(self):
+ write_entries(self.path, {"devicesmetainformation.auxiliary": devices_meta_xml([
+ ("DevA", [("App", "11223344-0000-0000-0000-000000000000", "55667788-0000-0000-0000-000000000000")])])})
+ text = buildinfo.render_info_text(buildinfo.read_info(self.path))
+ self.assertIn("CodeId 11223344-0000-0000-0000-000000000000", text)
+ self.assertIn("44332211", text)
+ self.assertIn("DataId 55667788", text)
+ self.assertIn("no compile time", text)
+
+ def test_missing_entry_is_a_note_and_exit_1(self):
+ write_entries(self.path, {})
+ info = buildinfo.read_info(self.path)
+ self.assertEqual(info.devices, [])
+ self.assertTrue(any("devicesmetainformation.auxiliary" in n for n in info.notes))
+ code, out, err = run_cli("info", self.path)
+ self.assertEqual(code, codesys_project.EXIT_PROBLEMS)
+ self.assertIn("no build identity", err)
+
+ def test_malformed_xml_is_a_note_not_a_crash(self):
+ write_entries(self.path, {"devicesmetainformation.auxiliary": b""})
+ info = buildinfo.read_info(self.path)
+ self.assertEqual(info.devices, [])
+ self.assertTrue(any("not well-formed" in n for n in info.notes))
+
+ def test_entity_declarations_are_refused_not_expanded(self):
+ bomb = (b']>'
+ b'&a;')
+ write_entries(self.path, {"devicesmetainformation.auxiliary": bomb})
+ info = buildinfo.read_info(self.path)
+ self.assertEqual(info.devices, [])
+ self.assertTrue(any("DOCTYPE" in n for n in info.notes))
+
+ def test_a_device_without_application_is_listed_with_none(self):
+ write_entries(self.path, {"devicesmetainformation.auxiliary": devices_meta_xml([("Lone", [])])})
+ info = buildinfo.read_info(self.path)
+ self.assertEqual(info.devices[0].applications, [])
+ self.assertIn("no application", buildinfo.render_info_text(info))
+
+ def test_application_without_code_id_keeps_the_field_empty(self):
+ xml = (b'D'
+ b'A')
+ write_entries(self.path, {"devicesmetainformation.auxiliary": xml})
+ app = buildinfo.read_info(self.path).devices[0].applications[0]
+ self.assertIsNone(app.code_id)
+ self.assertIn("CodeId -", buildinfo.render_info_text(buildinfo.read_info(self.path)))
+
+ def test_last_saved_is_the_newest_entry_whatever_the_order(self):
+ saves = [("2021-01-01-00-00-00", "2021-06-30-10-00-00", 7, "2.0"),
+ ("2020-01-01-00-00-00", "2020-02-02-12-00-00", 5, "1.0"),
+ ("2022-01-01-00-00-00", "2021-12-31-23-59-59", 3, "3.0")]
+ write_entries(self.path, {"engineeringtoolsversions.auxiliary": tools_versions_xml(saves, "Prod", "2.6")})
+ info = buildinfo.read_info(self.path)
+ self.assertEqual(info.last_saved, "2021-12-31 23:59:59")
+ self.assertEqual((info.save_counter, info.core_version), (3, "3.0"))
+ self.assertEqual((info.product, info.project_version), ("Prod", "2.6"))
+
+ def test_a_malformed_save_stamp_is_ignored(self):
+ saves = [("2020-01-01-00-00-00", "9999-bad", 5, "1.0"), ("2020-01-01-00-00-00", "2020-02-02-12-00-00", 4, "1.0")]
+ write_entries(self.path, {"engineeringtoolsversions.auxiliary": tools_versions_xml(saves)})
+ self.assertEqual(buildinfo.read_info(self.path).last_saved, "2020-02-02 12:00:00")
+
+ def test_a_non_numeric_save_counter_is_left_empty(self):
+ xml = (b'')
+ write_entries(self.path, {"engineeringtoolsversions.auxiliary": xml})
+ info = buildinfo.read_info(self.path)
+ self.assertEqual((info.last_saved, info.save_counter), ("2020-02-02 12:00:00", None))
+
+ def test_no_history_is_not_a_note(self):
+ write_entries(self.path, {"devicesmetainformation.auxiliary": devices_meta_xml([("D", [])])})
+ info = buildinfo.read_info(self.path)
+ self.assertIsNone(info.last_saved)
+ self.assertEqual(info.notes, [])
+ self.assertIn("last saved: not available", buildinfo.render_info_text(info))
+
+ def test_not_a_project_is_a_format_error(self):
+ bad = os.path.join(self.tmp.name, "bad.project")
+ Path(bad).write_bytes(b"not a zip")
+ with self.assertRaises(FormatError):
+ buildinfo.read_info(bad)
+ self.assertEqual(run_cli("info", bad)[0], codesys_project.EXIT_FATAL)
+
+ def test_json_output(self):
+ build_version(self.path, 2)
+ code, out, _ = run_cli("info", self.path, "--json")
+ data = json.loads(out)
+ self.assertEqual(code, 0)
+ app = data["devices"][0]["applications"][0]
+ self.assertEqual(app["code_id"], "aaaa0002-0000-0000-0000-000000000000")
+ self.assertEqual(app["code_id_short"], "aaaa0002")
+ self.assertEqual(app["code_id_byte_swapped"], "0200aaaa")
+ self.assertEqual(data["last_saved"], "2020-05-15 08:30:45")
+ self.assertEqual(data["notes"], [])
+
+
+class ObjectDiff(Versions):
+ def diff(self, name, *extra):
+ return run_cli("diff", self.old, self.new, name, *extra)
+
+ def test_ids_and_offsets_alone_produce_no_difference(self):
+ code, out, _ = self.diff("Shifted")
+ self.assertEqual(code, 0)
+ self.assertIn("no differences in the compared fields", out)
+ self.assertNotIn("@@", out)
+
+ def test_the_shifted_pair_really_differs_in_offsets_and_ids(self):
+ """Control for the test above: the two objects are not byte-identical or id-identical."""
+ a = project.load_project(self.old)
+ b = project.load_project(self.new)
+ ga = diff.graph_lines(a, a.objects[self.ids["shifted"]])
+ gb = diff.graph_lines(b, b.objects[self.ids2["shifted"]])
+ self.assertEqual(ga, gb)
+ from codesysproj.graph import decode_graph
+ ra = decode_graph(a.objects[self.ids["shifted"]], a.blobs[self.ids["shifted"]], a.strings)
+ rb = decode_graph(b.objects[self.ids2["shifted"]], b.blobs[self.ids2["shifted"]], b.strings)
+ self.assertNotEqual([i.offset for n in ra.networks for i in n.items], [i.offset for n in rb.networks for i in n.items])
+ self.assertNotEqual([i.ids for n in ra.networks for i in n.items], [i.ids for n in rb.networks for i in n.items])
+
+ def test_graph_changes_show_constant_modifier_comment_and_added_network(self):
+ code, out, _ = self.diff("Ladder")
+ self.assertEqual(code, 0)
+ self.assertRegex(out, r"(?m)^-.*IN '500'")
+ self.assertRegex(out, r"(?m)^\+.*IN '600'")
+ self.assertRegex(out, r"(?m)^-.*OUT/coil 'Lamp' type=BOOL mod=2 ~set")
+ self.assertRegex(out, r"(?m)^\+.*OUT/coil 'Lamp' type=BOOL mod=3 ~reset")
+ self.assertRegex(out, r"(?m)^-network comment='start the lamp'$")
+ self.assertRegex(out, r"(?m)^\+network comment='start the lamp slowly'$")
+ self.assertRegex(out, r"(?m)^\+network comment='added later'$")
+ self.assertRegex(out, r"(?m)^\+.*OUT 'Extra'")
+
+ def test_graph_diff_carries_no_ids_offsets_or_header_ids(self):
+ out = self.diff("Ladder")[1]
+ for leak in ("ids~", "id=", "5111", "6034", "9011", "9020"):
+ self.assertNotIn(leak, out)
+ self.assertNotRegex(out, r"(?m)^.\s*\d+ ") # no offset column
+
+ def test_graph_diff_says_what_it_cannot_see(self):
+ out = self.diff("Ladder")[1]
+ self.assertIn("wiring", out)
+ self.assertIn("not compared", out)
+
+ def test_unreadable_record_and_empty_slot_bytes_do_not_make_a_difference(self):
+ """An `unknown` item carries 16 raw bytes and an empty slot its id; neither is compared."""
+ from codesys_synth import graphical_payload, network, vi, qword
+ projects = []
+ for n, follow in enumerate((100, 7000)):
+ s = Synthetic()
+ bad = b"\x0f\xdf\x02" + vi(9999) + qword(follow)
+ s.add("G", guid(0), lambda g, s=s, bad=bad: make_object(graphical_payload(
+ s.t, g, network(s.t, "c", bad), "PROGRAM G\nVAR\nEND_VAR")))
+ path = os.path.join(self.tmp.name, "raw%d.project" % n)
+ s.write(path)
+ projects.append(project.load_project(path))
+ lines = [diff.graph_lines(p, p.objects[guid(1)]) for p in projects]
+ self.assertEqual(lines[0], lines[1])
+ self.assertTrue(any(line.startswith("UNKNOWN") for line in lines[0]))
+
+ def test_a_three_digit_constant_is_not_normalised_away(self):
+ a = project.load_project(self.old)
+ self.assertIn("IN '500'", "\n".join(diff.graph_lines(a, a.objects[self.ids["ladder"]])))
+
+ def test_unchanged_graph_in_a_changed_object_has_no_graph_section(self):
+ old = project.load_project(self.old)
+ new = project.load_project(self.new)
+ result = diff.object_diff(old, old.objects[self.ids["ladder"]], new, new.objects[self.ids2["ladder"]])
+ self.assertEqual([s.label for s in result.sections], ["graph"])
+
+ def test_st_declaration_and_implementation_are_diffed_separately(self):
+ code, out, _ = self.diff("Edited")
+ self.assertEqual(code, 0)
+ self.assertIn("--- OLD Edited declaration", out)
+ self.assertIn("+++ NEW Edited declaration", out)
+ self.assertRegex(out, r"(?m)^\+\ty: INT;$")
+ self.assertIn("--- OLD Edited implementation", out)
+ self.assertRegex(out, r"(?m)^-x := x \+ 1;$")
+ self.assertRegex(out, r"(?m)^\+x := x \+ 2;$")
+
+ def test_unified_diff_keeps_context_and_st_has_no_graph_note(self):
+ out = self.diff("Edited")[1]
+ self.assertIn("\n VAR\n \tx: INT;\n+\ty: INT;", out) # unchanged lines around the change
+ self.assertNotIn("wiring", out) # the limits note is for graphs
+
+ def test_identical_st_has_no_difference(self):
+ code, out, _ = self.diff("Same")
+ self.assertEqual(code, 0)
+ self.assertIn("no differences in the compared fields", out)
+
+ def test_rename_is_announced_in_an_object_diff(self):
+ out = run_cli("diff", self.old, self.new, self.ids["renamed"])[1]
+ self.assertIn("# renamed: 'OldName' -> 'NewName'", out)
+ self.assertIn("no differences in the compared fields", out)
+ self.assertNotIn("# renamed", self.diff("Edited")[1])
+
+ def test_a_graphical_slot_that_is_not_st_is_not_compared_as_implementation(self):
+ from codesys_synth import graphical_payload, network, slot
+ a, b = os.path.join(self.tmp.name, "gi1.project"), os.path.join(self.tmp.name, "gi2.project")
+ for path, text in ((a, "one"), (b, "two")):
+ s = Synthetic()
+ s.add("G", guid(0), lambda g, s=s, text=text: make_object(
+ graphical_payload(s.t, g, network(s.t, "c"), "PROGRAM G\nVAR\nEND_VAR") + slot(s.t, g, text, True)))
+ s.write(path)
+ rec = project.load_project(b).objects[guid(1)]
+ self.assertEqual((rec.kind, rec.implementation), ("graphical", "two")) # the slot exists and is read
+ self.assertIn("no differences in the compared fields", run_cli("diff", a, b, "G")[1])
+
+ def test_problem_in_the_new_object_sets_exit_1(self):
+ a, b = os.path.join(self.tmp.name, "pn1.project"), os.path.join(self.tmp.name, "pn2.project")
+ for path, twice in ((a, False), (b, True)):
+ s = Synthetic()
+ s.add("P", guid(0), lambda g, s=s, twice=twice: make_object(
+ st_payload(s.t, g, "PROGRAM P\nVAR\nEND_VAR", "x := 1;")
+ + (slot(s.t, g, "y := 2;", True) if twice else b"")))
+ s.write(path)
+ code, _, err = run_cli("diff", a, b, "P")
+ self.assertEqual(code, codesys_project.EXIT_PROBLEMS)
+ self.assertIn("PROBLEM", err)
+
+ def test_line_ending_only_change_is_not_a_difference(self):
+ old, new = os.path.join(self.tmp.name, "crlf1.project"), os.path.join(self.tmp.name, "crlf2.project")
+ for path, impl in ((old, "a := 1;\r\nb := 2;\r\n"), (new, "a := 1;\nb := 2;\n")):
+ s = Synthetic()
+ s.add("P", guid(0), lambda g, impl=impl, s=s: make_object(st_payload(s.t, g, "PROGRAM P\nVAR\nEND_VAR", impl)))
+ s.write(path)
+ out = run_cli("diff", old, new, "P")[1]
+ self.assertIn("no differences in the compared fields", out)
+
+ def test_name_missing_from_one_side_is_an_error(self):
+ for pou in ("Gone", "Fresh"):
+ code, _, err = self.diff(pou)
+ self.assertEqual(code, codesys_project.EXIT_FATAL)
+ self.assertIn("is not in NEW", self.diff("Gone")[2])
+ self.assertIn("is not in OLD", self.diff("Fresh")[2])
+
+ def test_a_guid_names_the_object_in_both_versions(self):
+ code, out, _ = run_cli("diff", self.old, self.new, self.ids["edited"])
+ self.assertEqual(code, 0)
+ self.assertIn("implementation", out)
+
+ def test_declaration_only_object_is_diffed(self):
+ out = self.diff("Globals")[1]
+ self.assertRegex(out, r"(?m)^\+\tmore: INT;$")
+ self.assertNotIn("implementation", out)
+
+ def test_object_that_is_not_comparable_is_an_error(self):
+ code, _, err = self.diff("Tasks")
+ self.assertEqual(code, codesys_project.EXIT_FATAL)
+ self.assertIn("kind 'other'", err)
+
+ def test_json_output(self):
+ code, out, _ = self.diff("Ladder", "--json")
+ data = json.loads(out)
+ self.assertEqual(code, 0)
+ self.assertEqual([s["label"] for s in data["sections"]], ["graph"])
+ self.assertTrue(any(line.startswith("+") for line in data["sections"][0]["diff"]))
+
+ def test_kind_change_diffs_the_union_of_sections(self):
+ a, b = os.path.join(self.tmp.name, "k1.project"), os.path.join(self.tmp.name, "k2.project")
+ from codesys_synth import graphical_payload, network
+ for path, graphical in ((a, False), (b, True)):
+ s = Synthetic()
+ decl = "PROGRAM K\nVAR\nEND_VAR"
+ s.add("K", guid(0), lambda g, graphical=graphical, s=s: make_object(
+ graphical_payload(s.t, g, network(s.t, "c"), decl) if graphical else st_payload(s.t, g, decl, "x := 1;")))
+ s.write(path)
+ out = run_cli("diff", a, b, "K")[1]
+ self.assertIn("kind changed: st -> graphical", out)
+ self.assertIn("implementation", out)
+ self.assertIn("graph", out)
+
+ def test_decode_notes_set_exit_1(self):
+ """A recognised record that cannot be read is reported, and the exit code says so."""
+ s = Synthetic()
+ from codesys_synth import element, graphical_payload, network, vi
+ bad = b"\x0f\xdf\x02" + vi(9999) # element whose name index is not in the string table
+ for name in ("n1.project", "n2.project"):
+ s = Synthetic()
+ s.add("G", guid(0), lambda g, s=s: make_object(graphical_payload(
+ s.t, g, network(s.t, "c", bad), "PROGRAM G\nVAR\nEND_VAR")))
+ s.write(os.path.join(self.tmp.name, name))
+ code, out, err = run_cli("diff", os.path.join(self.tmp.name, "n1.project"),
+ os.path.join(self.tmp.name, "n2.project"), "G")
+ self.assertEqual(code, codesys_project.EXIT_PROBLEMS)
+ self.assertIn("could not be decoded", err)
+ code, _, err = run_cli("diff", os.path.join(self.tmp.name, "n1.project"), os.path.join(self.tmp.name, "n2.project"))
+ self.assertEqual(code, codesys_project.EXIT_PROBLEMS)
+ self.assertIn("could not be decoded", err)
+
+
+class ProjectDiff(Versions):
+ def lines(self):
+ return run_cli("diff", self.old, self.new)[1].splitlines()
+
+ def test_added_removed_and_changed_are_listed_by_guid(self):
+ code, out, _ = run_cli("diff", self.old, self.new)
+ self.assertEqual(code, 0)
+ self.assertRegex(out, r"(?m)^\+ added +%s +st +.*Fresh$" % self.ids2["fresh"])
+ self.assertRegex(out, r"(?m)^- removed +%s +st +.*Gone$" % self.ids["gone"])
+ self.assertRegex(out, r"(?m)^~ changed +%s +st +.*Edited +\[declaration, implementation\]$" % self.ids["edited"])
+ self.assertRegex(out, r"(?m)^~ changed +%s +graphical +.*Ladder +\[graph\]$" % self.ids["ladder"])
+ self.assertRegex(out, r"(?m)^~ changed +%s +gvl +.*Globals +\[declaration\]$" % self.ids["globals"])
+
+ def test_unchanged_objects_are_not_listed(self):
+ out = run_cli("diff", self.old, self.new)[1]
+ for name in ("Same", "Shifted"):
+ self.assertNotRegex(out, r"(?m)^[~+-] .*\b%s$" % name)
+
+ def test_rename_with_the_same_text_is_a_change_with_both_names(self):
+ out = run_cli("diff", self.old, self.new)[1]
+ self.assertRegex(out, r"(?m)^~ changed +%s +st +.*NewName +\[renamed from 'OldName'\]$" % self.ids["renamed"])
+
+ def test_objects_that_are_not_compared_are_counted_not_hidden(self):
+ out = run_cli("diff", self.old, self.new)[1]
+ self.assertRegex(out, r"(?m)^= \d+ unchanged, 4 not compared \(kind other or unreadable\)")
+
+ def test_summary_counts_add_up(self):
+ a, b = project.load_project(self.old), project.load_project(self.new)
+ result = diff.project_diff(a, b)
+ self.assertEqual(len(result.added), 1)
+ self.assertEqual(len(result.removed), 1)
+ self.assertEqual(len(result.changed), 4) # Edited, Ladder, Renamed, Globals
+ both = set(a.objects) & set(b.objects)
+ self.assertEqual(len(both), result.unchanged + result.not_compared + len(result.changed))
+
+ def test_same_project_against_itself_lists_nothing(self):
+ out = run_cli("diff", self.old, self.old)[1]
+ self.assertNotRegex(out, r"(?m)^[~+-] ")
+
+ def test_unreadable_on_one_side_is_not_compared_not_changed(self):
+ a, b = os.path.join(self.tmp.name, "u1.project"), os.path.join(self.tmp.name, "u2.project")
+ for path, payload in ((a, lambda s: lambda g: make_object(st_payload(s.t, g, "PROGRAM U\nVAR\nEND_VAR", "x := 1;"))),
+ (b, lambda s: lambda g: b"short")):
+ s = Synthetic()
+ s.add("U", guid(0), payload(s))
+ s.write(path)
+ out = run_cli("diff", a, b)[1]
+ self.assertNotRegex(out, r"(?m)^~ changed")
+ self.assertIn("1 not compared", out)
+
+ def test_an_object_moved_to_another_parent_is_a_change(self):
+ a, b = os.path.join(self.tmp.name, "m1.project"), os.path.join(self.tmp.name, "m2.project")
+ for path, parent in ((a, 1), (b, 2)):
+ s = Synthetic()
+ s.add("Left", guid(0), make_object(b"\x0f\x01"))
+ s.add("Right", guid(0), make_object(b"\x0f\x01"))
+ s.add("Mover", guid(parent), lambda g, s=s: make_object(st_payload(s.t, g, "PROGRAM M\nVAR\nEND_VAR", "x := 1;")))
+ s.write(path)
+ out = run_cli("diff", a, b)[1]
+ self.assertRegex(out, r"(?m)^~ changed +\S+ +st +Right/Mover +\[moved from 'Left'\]$")
+ self.assertNotIn("renamed", out)
+
+ def test_ambiguous_name_is_refused_with_the_candidates(self):
+ from codesys_synth import build_standard
+ std = os.path.join(self.tmp.name, "std.project")
+ build_standard(std)
+ code, _, err = run_cli("diff", std, std, "Main")
+ self.assertEqual(code, codesys_project.EXIT_FATAL)
+ self.assertIn("ambiguous", err)
+
+ def test_json_output(self):
+ code, out, _ = run_cli("diff", self.old, self.new, "--json")
+ data = json.loads(out)
+ self.assertEqual(code, 0)
+ self.assertEqual([c["name"] for c in data["added"]], ["Fresh"])
+ self.assertEqual([c["name"] for c in data["removed"]], ["Gone"])
+ self.assertEqual(sorted(c["name"] for c in data["changed"]), ["Edited", "Globals", "Ladder", "NewName"])
+ self.assertEqual(data["not_compared"], 4)
+
+ def test_problem_in_an_object_sets_exit_1(self):
+ a = os.path.join(self.tmp.name, "pr1.project")
+ s = Synthetic()
+ s.add("Broken", guid(0), None) # a meta entry without an object: reported as a problem
+ s.write(a)
+ code, _, err = run_cli("diff", a, a)
+ self.assertEqual(code, codesys_project.EXIT_PROBLEMS)
+ self.assertIn("PROBLEM", err)
+
+
+class Xref(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ cls.tmp = tempfile.TemporaryDirectory()
+ cls.path = os.path.join(cls.tmp.name, "x.project")
+ cls.ids = build_xref(cls.path)
+ cls.project = project.load_project(cls.path)
+
+ @classmethod
+ def tearDownClass(cls):
+ cls.tmp.cleanup()
+
+ def hits(self, query, name=None):
+ result = xref.find_references(self.project, query)
+ return [(h.name, h.where, h.role) for h in result.hits if name in (None, h.name)]
+
+ def test_st_roles_by_line(self):
+ got = self.hits("counter", "Writer")
+ self.assertIn(("Writer", "implementation:3", "write"), got)
+ self.assertIn(("Writer", "implementation:3", "read"), got)
+ self.assertIn(("Writer", "implementation:4", "write"), got) # Gvl.counter := 2
+ self.assertIn(("Writer", "implementation:5", "read"), got) # arr[counter] := 0: an index is a read
+ self.assertIn(("Writer", "implementation:7", "arg"), got) # IN := COUNTER > 3, any case
+ self.assertIn(("Writer", "implementation:7", "write"), got) # Q => counter
+ self.assertIn(("Writer", "implementation:9", "write"), got) # FOR counter := ...
+ self.assertIn(("Writer", "implementation:10", "read"), got) # Fb.Method(counter)
+
+ def test_comments_strings_and_longer_names_do_not_match(self):
+ lines = {w for _, w, _ in self.hits("counter", "Writer")}
+ for noise in ("implementation:1", "implementation:2", "implementation:6", "implementation:8"):
+ self.assertNotIn(noise, lines) # comment, block comment, string, counter_total
+
+ def test_assignment_target_is_not_also_a_read(self):
+ on_line_4 = [r for _, w, r in self.hits("counter", "Writer") if w == "implementation:4"]
+ self.assertEqual(on_line_4, ["write"])
+
+ def test_the_array_is_written_and_its_index_read(self):
+ got = self.hits("arr", "Writer")
+ self.assertEqual(got, [("Writer", "implementation:5", "write")])
+
+ def test_call_and_formal_parameter(self):
+ got = self.hits("Timer1", "Writer")
+ self.assertIn(("Writer", "implementation:7", "call"), got)
+ self.assertIn(("Writer", "declaration:4", "declared"), got) # `timer1 : TON;` any case
+ self.assertEqual([r for _, w, r in self.hits("IN", "Writer") if w == "implementation:7"], ["param"])
+ self.assertEqual([r for _, w, r in self.hits("Q", "Writer") if w == "implementation:7"], ["param"]) # Q => ...
+
+ def test_qualified_query_matches_the_qualified_path_only(self):
+ got = self.hits("Gvl.counter", "Writer")
+ self.assertIn(("Writer", "implementation:4", "write"), got)
+ self.assertNotIn(("Writer", "implementation:3", "write"), got) # bare `counter`
+ self.assertNotIn("implementation:11", {w for _, w, _ in got}) # `Gvl.other`: same first segment only
+
+ def test_declaration_roles(self):
+ got = self.hits("counter")
+ self.assertIn(("Gvl", "declaration:1", "declared"), got) # after `VAR_GLOBAL RETAIN` on the same line
+ self.assertIn(("Writer", "declaration:3", "read"), got) # initialiser `:= Gvl.counter`
+ self.assertIn(("Lad", "declaration:3", "declared"), got)
+ self.assertEqual([h for h in self.hits("spare") if h[0] == "Gvl"], [("Gvl", "declaration:2", "declared")])
+ self.assertEqual([h for h in self.hits("other", "Gvl")], [("Gvl", "declaration:2", "declared")])
+ self.assertEqual(self.hits("lone"), [("Gvl", "declaration:3", "declared")]) # `lone AT %MX0.0 : BOOL`
+
+ def test_type_role(self):
+ self.assertIn(("Writer", "declaration:4", "type"), self.hits("TON"))
+
+ def test_qualified_declaration_needs_the_owning_object_name(self):
+ self.assertIn(("Gvl", "declaration:1", "declared"), self.hits("Gvl.counter"))
+ self.assertNotIn(("Lad", "declaration:3", "declared"), self.hits("Gvl.counter"))
+ self.assertNotIn(("Gvl", "declaration:1", "declared"), self.hits("Other.counter"))
+
+ def test_graphical_roles(self):
+ got = self.hits("counter", "Lad")
+ self.assertIn(("Lad", "network 1", "write"), got) # OUT 'Gvl.counter'
+ self.assertIn(("Lad", "network 1", "read"), got) # IN 'arr[counter]': index
+ self.assertEqual(self.hits("other", "Lad"), [("Lad", "network 1", "read")]) # IN
+ self.assertEqual(self.hits("arr", "Lad"), [("Lad", "network 1", "read"), ("Lad", "network 2", "write")])
+ self.assertEqual(self.hits("idx", "Lad"), [("Lad", "network 2", "read")]) # index of an OUT is a read
+
+ def test_graphical_box_type_and_instance(self):
+ self.assertIn(("Lad", "network 1", "type"), self.hits("TON", "Lad"))
+ self.assertIn(("Lad", "network 1", "call"), self.hits("timer1", "Lad"))
+
+ def test_literals_and_addresses_are_not_names(self):
+ code = "x := 16#FF + T#5s + 1.5e3 + REAL#1.0 + %IX0.0 + a[1..n] + b.c + ptr^.field + d._e;"
+ self.assertEqual([".".join(p.segments) for p in stscan.scan_paths(code)],
+ ["x", "a", "n", "b.c", "ptr.field", "d._e"])
+
+ def test_unknown_name_has_no_hits(self):
+ self.assertEqual(xref.find_references(self.project, "nothing_here").hits, [])
+
+ def test_objects_not_searched_are_counted(self):
+ result = xref.find_references(self.project, "counter")
+ self.assertEqual(result.not_searched, 4) # Dev, Plc Logic, Application, Tasks
+ self.assertEqual(result.searched, 3) # Gvl, Writer, Lad
+
+ def test_text_output_marks_heuristics_and_limits(self):
+ code, out, _ = run_cli("xref", self.path, "counter")
+ self.assertEqual(code, 0)
+ self.assertIn("heuristic", out)
+ self.assertIn("no scope resolution", out)
+ self.assertIn("not searched", out)
+ self.assertRegex(out, r"Writer +\(st\) +implementation:3 +write")
+ self.assertRegex(out, r"(?m)^\d+ reference\(s\) in 3 object\(s\)")
+
+ def test_json_output(self):
+ code, out, _ = run_cli("xref", self.path, "counter", "--json")
+ data = json.loads(out)
+ self.assertEqual(code, 0)
+ self.assertEqual(data["query"], "counter")
+ self.assertTrue(all(set(h) >= {"guid", "name", "kind", "where", "role", "text"} for h in data["hits"]))
+ self.assertEqual(data["not_searched"], 4)
+
+ def test_a_damaged_object_sets_exit_1(self):
+ path = os.path.join(self.tmp.name, "dmg.project")
+ s = Synthetic()
+ s.add("Broken", guid(0), None)
+ s.write(path)
+ self.assertEqual(run_cli("xref", path, "x")[0], codesys_project.EXIT_PROBLEMS)
+
+ def test_empty_query_is_an_error(self):
+ self.assertEqual(run_cli("xref", self.path, " ")[0], codesys_project.EXIT_FATAL)
+ self.assertEqual(run_cli("xref", self.path, "a..b")[0], codesys_project.EXIT_FATAL)
+ self.assertEqual(run_cli("xref", self.path, "5")[0], codesys_project.EXIT_FATAL)
+
+ def test_unreadable_graph_records_are_reported(self):
+ """A reference could hide in a record the decoder cannot read: say so."""
+ from codesys_synth import graphical_payload, network, vi
+ path = os.path.join(self.tmp.name, "u.project")
+ s = Synthetic()
+ s.add("G", guid(0), lambda g: make_object(graphical_payload(
+ s.t, g, network(s.t, "c", b"\x0f\xdf\x02" + vi(9999)), "PROGRAM G\nVAR\nEND_VAR")))
+ s.write(path)
+ code, out, err = run_cli("xref", path, "anything")
+ self.assertEqual(code, codesys_project.EXIT_PROBLEMS)
+ self.assertIn("could not be decoded", out + err)
+
+
+class Headers(unittest.TestCase):
+ """A declaration whose header carries a `:` (a function's return type, a type's definition)."""
+
+ @staticmethod
+ def hits(decl, query, kind=project.KIND_GVL, name="Owner"):
+ rec = project.ObjectRecord(guid=guid(1), name=name, kind=kind, declaration=decl,
+ implementation="" if kind == project.KIND_ST else None)
+ proj = project.Project(path="", strings={}, objects={rec.guid: rec}, blobs={})
+ return [(h.where, h.role) for h in xref.find_references(proj, query).hits]
+
+ def test_return_type_is_a_type_and_the_first_variable_is_declared(self):
+ decl = "FUNCTION Func : BOOL\nVAR_INPUT\n\tx : INT;\nEND_VAR"
+ self.assertEqual(self.hits(decl, "x"), [("declaration:3", "declared")])
+ self.assertEqual(self.hits(decl, "BOOL"), [("declaration:1", "type")])
+ self.assertEqual(self.hits(decl, "INT"), [("declaration:3", "type")])
+ self.assertEqual(self.hits(decl, "FUNCTION"), [])
+ self.assertEqual(self.hits(decl, "Func"), [("declaration:1", "declared")])
+
+ def test_every_header_keyword_is_a_header(self):
+ for keyword in ("FUNCTION", "FUNCTION_BLOCK", "METHOD", "PROPERTY", "PROGRAM", "INTERFACE", "ACTION",
+ "TRANSITION", "function", "Method"):
+ decl = "%s Head : Ret\nVAR\n\tmember : INT;\nEND_VAR" % keyword
+ with self.subTest(keyword):
+ self.assertEqual(self.hits(decl, "member"), [("declaration:3", "declared")])
+ self.assertEqual(self.hits(decl, "Ret"), [("declaration:1", "type")])
+ self.assertEqual(self.hits(decl, keyword), [])
+
+ def test_header_without_a_colon_leaves_the_variables_alone(self):
+ decl = "PROGRAM Prog\nVAR\n\tmember : INT;\nEND_VAR"
+ self.assertEqual(self.hits(decl, "member"), [("declaration:3", "declared")])
+ self.assertEqual(self.hits(decl, "Prog"), [("declaration:1", "declared")])
+ self.assertEqual(self.hits("PROGRAM Prog VAR x : INT; END_VAR", "x"), [("declaration:1", "declared")])
+
+ def test_struct_members_after_a_type_header(self):
+ multi = "TYPE Shape :\nSTRUCT\n\tcounter : INT;\n\tarea : REAL;\nEND_STRUCT\nEND_TYPE"
+ self.assertEqual(self.hits(multi, "counter", project.KIND_DUT), [("declaration:3", "declared")])
+ self.assertEqual(self.hits(multi, "area", project.KIND_DUT), [("declaration:4", "declared")])
+ self.assertEqual(self.hits(multi, "Shape", project.KIND_DUT), [("declaration:1", "declared")])
+ self.assertEqual(self.hits(multi, "STRUCT", project.KIND_DUT), []) # a keyword, not a type
+ one = "TYPE Pair : STRUCT counter : INT; second : INT; END_STRUCT END_TYPE"
+ self.assertEqual(self.hits(one, "counter", project.KIND_DUT), [("declaration:1", "declared")])
+ self.assertEqual(self.hits(one, "second", project.KIND_DUT), [("declaration:1", "declared")])
+ both = "TYPE Pair : STRUCT\n\tcounter : INT;\nEND_STRUCT\nEND_TYPE" # STRUCT comes before the line end
+ self.assertEqual(self.hits(both, "STRUCT", project.KIND_DUT), [])
+ self.assertEqual(self.hits(both, "counter", project.KIND_DUT), [("declaration:2", "declared")])
+
+ def test_header_name_of_another_object_needs_the_qualifier(self):
+ decl = "FUNCTION Func : BOOL\nVAR_INPUT\n\tx : INT;\nEND_VAR"
+ self.assertEqual(self.hits(decl, "Other.Func"), [])
+ self.assertEqual(self.hits(decl, "Owner.Func"), [("declaration:1", "declared")])
+
+ def test_a_variable_named_like_a_header_keyword_is_still_a_variable(self):
+ decl = "VAR_GLOBAL\n\taction : BOOL;\n\ttype : INT;\n\tmethod: INT;\nEND_VAR"
+ self.assertEqual(self.hits(decl, "action"), [("declaration:2", "declared")])
+ self.assertEqual(self.hits(decl, "type"), [("declaration:3", "declared")])
+ self.assertEqual(self.hits(decl, "method"), [("declaration:4", "declared")])
+ one = "VAR_GLOBAL a : INT; type : INT; action : BOOL; END_VAR" # later chunks that start mid-line
+ self.assertEqual(self.hits(one, "type"), [("declaration:1", "declared")])
+ self.assertEqual(self.hits(one, "action"), [("declaration:1", "declared")])
+
+ def test_a_header_alone_on_one_line_still_has_its_type(self):
+ self.assertEqual(self.hits("FUNCTION F : BOOL", "BOOL"), [("declaration:1", "type")])
+ self.assertEqual(self.hits("FUNCTION F : BOOL", "FUNCTION"), [])
+
+ def test_a_declaration_with_no_colon_at_all(self):
+ self.assertEqual(self.hits("PROGRAM Prog", "x"), [])
+ self.assertEqual(self.hits("PROGRAM Prog", "Prog"), [("declaration:1", "declared")])
+ self.assertEqual(self.hits("", "x"), [])
+
+ def test_text_after_the_last_declaration_is_not_a_name(self):
+ self.assertEqual(self.hits("VAR\n\tx : INT;\nEND_VAR", "END_VA"), [])
+
+
+class StripNoise(unittest.TestCase):
+ def test_comments_pragmas_and_strings_are_blanked_with_positions_kept(self):
+ text = "a (* b\n (* nested *) c *) d // e\n{attr 'x'} f 'g $' h' i \"j\" k"
+ out = stscan.strip_noise(text)
+ self.assertEqual(len(out), len(text))
+ self.assertEqual(out.count("\n"), 2) # one inside the block comment, one after the line comment
+ self.assertEqual(out.split(), ["a", "d", "f", "i", "k"])
+
+ def test_unterminated_comment_and_string_blank_to_the_end(self):
+ self.assertEqual(stscan.strip_noise("a (* b").split(), ["a"])
+ self.assertEqual(stscan.strip_noise("a 'b").split(), ["a"])
+ self.assertEqual(stscan.strip_noise("a {b").split(), ["a"])
+
+ def test_comment_marker_inside_a_string_is_text(self):
+ self.assertEqual(stscan.strip_noise("a '(* x' b").split(), ["a", "b"])
+
+
+if __name__ == "__main__":
+ unittest.main()