diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 6fb6ff6..bb7ff5a 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -8,7 +8,7 @@ "plugins": [ { "name": "plc-programming", - "description": "Design, write, convert, review and debug PLC programs: IEC 61131-3 (ST, LD, FBD, SFC, IL), PLCopen XML / IEC 61131-10, Siemens TIA (SCL, SimaticML, SIMATIC SD), Schneider (Machine Expert, Control Expert), CODESYS, TwinCAT, ABB Freelance, Rockwell L5X/L5K, IEC 61499, AutomationML, DEXPI, PackML/ISA-88, OPC UA; with a stdlib-only structural validator, an edit-time validation hook, and an offline reader for CODESYS / Machine Expert .project files.", + "description": "Design, write, convert, review and debug PLC programs: IEC 61131-3 (ST, LD, FBD, SFC, IL), PLCopen XML / IEC 61131-10, Siemens TIA (SCL, SimaticML, SIMATIC SD), Schneider (Machine Expert, Control Expert), CODESYS, TwinCAT, ABB Freelance, Rockwell L5X/L5K, IEC 61499, AutomationML, DEXPI, PackML/ISA-88, OPC UA; with a stdlib-only structural validator, an edit-time validation hook, an offline reader for CODESYS / Machine Expert .project files, and an offline decoder for WinCC Tag Logging fast-archive blocks.", "source": "./" } ] diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index 79560aa..e14c0cf 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "$schema": "https://anthropic.com/claude-code/plugin.schema.json", "name": "plc-programming", - "version": "0.2.2", + "version": "0.3.0", "description": "PLC programming across IEC 61131-3 (ST, LD, FBD, SFC, IL), PLCopen XML / IEC 61131-10, Siemens TIA (SCL, SimaticML, SIMATIC SD), Schneider (Machine Expert, Control Expert), CODESYS, TwinCAT, ABB Freelance, Rockwell L5X/L5K, IEC 61499, AutomationML, DEXPI, PackML/ISA-88 and OPC UA: design, write, convert, review and debug programs, with a stdlib-only structural validator.", "author": { "name": "SmailG", diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7e1abd8..8431477 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -13,13 +13,14 @@ privately, as described in [SECURITY.md](SECURITY.md). Pull requests use the tem | `hooks/hooks.json` | PostToolUse hook that runs the validator on every file Claude writes or edits | | `scripts/plc_validate.py` | Validator CLI and hook entry point | | `scripts/codesys_project.py` | Offline reader for CODESYS 3.5 / Machine Expert `.project` files (`list`, `st`, `dump-st`, `graph`, `info`, `diff`, `xref`); `scripts/codesysproj/` holds `binary.py` (varint, string table), `project.py` (objects, kinds, text), `graph.py` (LD/FBD records) and `graphview.py` (their text and JSON), `export.py` (`dump-st`), `buildinfo.py` (`info`), `diff.py` (`diff`), `xref.py` and `stscan.py` (`xref`) | +| `scripts/wincc_binvalues.py` | Offline decoder and CLI for WinCC 7 Tag Logging fast-archive blocks (`TagCompressed.BinValues` to CSV) | | `scripts/plccheck/` | One module per format (`st.py`, `plcopen.py`, `simaticml.py`, `twincat.py`, `logix.py`, `schneider.py`, `iec61499.py`, `aml.py`, `dexpi.py`, `tabular.py`); `findings.py` holds the finding type, `xmlutil.py` the line-numbered XML parser | -| `tests/` | `test_st.py`, `test_formats.py` (validator rules), `test_codesys_project.py` and `test_codesys_tools.py` (the `.project` reader and its `info`, `diff` and `xref`, on synthetic projects only), `test_codesys_hardening.py` (hostile and damaged `.project` input and the reader's CLI contracts, synthetic projects only), `test_linear_xml.py`, `test_linear_aml.py` and `test_linear_text.py` (CPU-time scaling and ordering of the format checkers on crafted input; `linear_support.py` holds their ratio helper and XML builders), `test_skills.py` (frontmatter, templates, every `iecst` block lints clean) | +| `tests/` | `test_st.py`, `test_formats.py` (validator rules), `test_codesys_project.py` and `test_codesys_tools.py` (the `.project` reader and its `info`, `diff` and `xref`, on synthetic projects only), `test_codesys_hardening.py` (hostile and damaged `.project` input and the reader's CLI contracts, synthetic projects only), `test_linear_xml.py`, `test_linear_aml.py` and `test_linear_text.py` (CPU-time scaling and ordering of the format checkers on crafted input; `linear_support.py` holds their ratio helper and XML builders), `test_wincc_binvalues.py` (the WinCC archive decoder and its CLI, on synthetic blocks built by an encoder inside the test), `test_skills.py` (frontmatter, templates, every `iecst` block lints clean) | | `evals/` | `claude plugin eval` cases comparing the plugin against a no-plugin baseline | ## Rules -- **The validator and the `.project` reader use only the Python standard library** and must +- **The validator, the `.project` reader and the WinCC archive decoder use only the Python standard library** and must run on Python 3.9 (CI runs 3.9 and 3.12). Optional tools such as `xmllint` or `lxml` are allowed only behind `--xsd`. - **Every validator rule needs two tests:** an input it must catch and a valid control it must accept. A rule tested only on bad input cannot show it is not firing on everything. @@ -34,6 +35,9 @@ privately, as described in [SECURITY.md](SECURITY.md). Pull requests use the tem - Do not bundle vendor schemas or manuals whose licence does not clearly permit redistribution. - **Test fixtures for the `.project` reader are synthetic.** Never commit a real `.project`, or names, text or guids taken from one: it is someone's program. +- **Test fixtures for the WinCC archive decoder are synthetic too.** Never commit real block + bytes, decoded values, tag names, segment or project names or host names from an archive: it is + someone's process data. Build blocks with the encoder in the test. ## Tests diff --git a/README.md b/README.md index c986d4b..1aeb4bc 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,7 @@ git clone https://github.com/SmailG/plc-programming.git .agents/plugins/plc-prog | `schneider` skill | automatic | Machine Expert (M241/M251/M262), Control Expert (M340/M580), Automation Expert, Modbus, EtherNet/IP, OPC UA | | `iec-61131-3` skill | automatic | Ed.2–Ed.4, the five languages, POUs, OOP, data types, the standard library, PLCopen guidelines, Motion and Safety | | `exchange-formats` skill | automatic | PLCopen XML (TC6 v2.0/v2.01), IEC 61131-10, AutomationML (IEC 62714), DEXPI / ISO 15926 | -| `other-platforms` skill | automatic | Beckhoff TwinCAT, CODESYS (incl. reading a binary `.project` offline: ST, build identity, version diff, cross-reference), ABB Freelance (PRT/CSV) / AC500 / 800xA, Rockwell L5X/L5K | +| `other-platforms` skill | automatic | Beckhoff TwinCAT, CODESYS (incl. reading a binary `.project` offline: ST, build identity, version diff, cross-reference), ABB Freelance (PRT/CSV) / AC500 / 800xA, Rockwell L5X/L5K, Siemens WinCC 7 Tag Logging fast-archive blocks to CSV (offline decoder) | | `iec-61499` skill | automatic | event-driven FBs, ECC, 4diac file formats | | `packml-isa88-opcua` skill | automatic | PackML states and PackTags, ISA-88 / IEC 61512, OPC UA (OPC 30000/30001/30050) | | `/plc-programming:new-pou` | user only | scaffold a POU for a chosen platform and format | @@ -105,6 +105,22 @@ not crashed on. Prefer a PLCopen XML export from the IDE when you can get one, and verify graphical logic in the IDE. Details: [codesys-project-files.md](skills/other-platforms/references/codesys-project-files.md). +## The WinCC Tag Logging archive decoder + +```bash +python3 scripts/wincc_binvalues.py dump.txt > tag.csv # or one block as hex on standard input +``` + +Decodes the fast-archive blocks (`TagCompressed.BinValues`) of a Siemens WinCC 7 Tag Logging +segment to CSV (`timestamp,value,quality`) without WinCC (standard library only, Python 3.9+). +**The format is undocumented**: the decoder is reverse-engineered on WinCC 7.5 SP2 and untested +on other versions. It raises an error for a block that fails its checks; it does not check some +header bytes, and a damaged value inside the stream can still decode. The compressed archive +type is not decoded and the signedness of the 32-bit integer code is a guess. A block that fails +is reported by line and the others are still written. Attach copies of the segments to a +separate SQL Server instance, never a live WinCC. The workflow, the query and what is established: +[wincc-tag-logging-archive.md](skills/other-platforms/references/wincc-tag-logging-archive.md). + ## Tests and evals ```bash @@ -114,7 +130,7 @@ claude plugin eval . --trust-plugin --allow-tools Write Edit "Bash(python3 *)" - - **Unit tests:** every validator rule is exercised by an input it must catch and by a valid control it must pass. The `.project` reader is tested on synthetic projects built in - the test, never on a real program. Every shipped template and every `iecst` code block in the + the test, never on a real program, and the WinCC decoder on synthetic blocks, never on real archive data. Every shipped template and every `iecst` code block in the skills must lint clean. - **Evals:** they compare the plugin against a no-plugin baseline on edition facts, SFC semantics, a classic scan-cycle bug, PLCopen XML generation, SimaticML namespaces, diff --git a/plugin.json b/plugin.json index e4b35fd..e8e3825 100644 --- a/plugin.json +++ b/plugin.json @@ -1,7 +1,7 @@ { "name": "plc-programming", "displayName": "PLC Programming", - "version": "0.2.2", + "version": "0.3.0", "description": "PLC programming across IEC 61131-3 (ST, LD, FBD, SFC, IL), PLCopen XML / IEC 61131-10, Siemens TIA, Schneider, CODESYS, TwinCAT, ABB Freelance, Rockwell L5X/L5K, IEC 61499, AutomationML, DEXPI, PackML/ISA-88 and OPC UA with a stdlib-only structural validator.", "author": { "name": "SmailG", diff --git a/scripts/wincc_binvalues.py b/scripts/wincc_binvalues.py new file mode 100644 index 0000000..2284f3e --- /dev/null +++ b/scripts/wincc_binvalues.py @@ -0,0 +1,280 @@ +#!/usr/bin/env python3 +"""Decode WinCC Tag Logging fast-archive blocks (`TagCompressed.BinValues`) offline. + + wincc_binvalues.py [DUMP] + +DUMP is a file, or `-` or nothing for standard input. A file is read as UTF-8 (a byte order mark +is accepted; a UTF-16 file is refused with exit 2). Each line is either one block as hex, or +four `|`-separated fields as `sqlcmd` prints them, the last being the hex (the first three are +ignored; the block's own header carries the start time). Output is CSV on standard output: +`timestamp,value,quality`, timestamps in UTC to the millisecond, quality as the decimal OPC +quality byte (192 = 0xC0 = good, 8 = 0x08 = bad / not connected). + +The format is undocumented. It was reverse-engineered on WinCC 7.5 SP2 fast archives and is +untested on other versions. The decoder raises DecodeError on a block that fails its checks +(checksum, length, sample count and period, value code, chunk headers, the stream's own +consistency) rather than guessing. It does not check header bytes 1-3, 5-6 and 25-36, the value of +a quality byte or the daylight-saving bit, and a damaged value inside the stream can decode. +Established, guessed and undecoded parts are listed in +skills/other-platforms/references/wincc-tag-logging-archive.md. + +As a library: + + from wincc_binvalues import decode_block + rows = decode_block(binvalues_bytes) # [(datetime, value, quality), ...] + +Exit: 0 clean, 1 some blocks did not decode or none were found (the others are still written), +2 usage or unreadable input. Standard library only; Python 3.9 or later. +""" +import argparse +import csv +import datetime +import struct +import sys + +HEADER_LEN = 37 # fixed byte header before the bit stream +MRU_SIZE = 19 # history depth (current + 18 back); index 19 is never referenced +OLE_EPOCH = datetime.datetime(1899, 12, 30) + +# byte 4 of the header -> (kind, width in bits, signed). 'f' = IEEE float32 little-endian. +# Widths are established (bit accounting). Signedness of 4-7 follows from observed values; +# signedness of 8 is a guess (no value >= 2**31 was seen). +VALUE_CODES = { + 3: ('f', 32, True), + 4: ('i', 16, True), + 5: ('i', 8, True), + 6: ('i', 8, False), + 7: ('i', 16, False), + 8: ('i', 32, True), +} + + +class DecodeError(ValueError): + pass + + +class _Bits: + def __init__(self, data): + self.s = ''.join(f'{x:08b}' for x in data) + self.p = 0 + + def take(self, n): + if self.p + n > len(self.s): + raise DecodeError(f'read past end at bit {self.p} (+{n})') + v = self.s[self.p:self.p + n] + self.p += n + return v + + def peek(self, n): + return self.s[self.p:self.p + n] + + +def _raw(bitstr, width): + return int(bitstr, 2).to_bytes(width // 8, 'big') # bytes in stream order + + +def _value(raw, kind, signed): + if kind == 'f': + return struct.unpack(' MRU_SIZE: + hist.pop() + + if bs.take(1) != '0': + raise DecodeError('stream does not start with a value token') + cur = _raw(bs.take(width), width) + push(cur) + vals = [cur] + left, q = _chunk_header(bs) + chunks = [(left, q)] + counts = {'same': 0, 'new': 1, 'ref': 0, 'chunks': 1} + while len(vals) < n: + if left == 0: + left, q = _chunk_header(bs) + chunks.append((left, q)) + counts['chunks'] += 1 + continue + if bs.peek(2) == '11': + bs.take(2) + counts['same'] += 1 + elif bs.peek(1) == '0': + bs.take(1) + cur = _raw(bs.take(width), width) + push(cur) + counts['new'] += 1 + else: + bs.take(2) + c = int(bs.take(2), 2) + idx = c if c else 4 + int(bs.take(4), 2) + if idx >= len(hist): + raise DecodeError(f'history index {idx} beyond {len(hist)} at sample {len(vals)}') + cur = hist[idx] + push(cur) + counts['ref'] += 1 + vals.append(cur) + left -= 1 + if left == 0: + # the chunk counts sum to n: a block whose last chunk filled up exactly ends with + # an empty chunk header (m = 1, no tokens) + left, q = _chunk_header(bs) + chunks.append((left, q)) + counts['chunks'] += 1 + pad = bs.s[bs.p:] + if len(pad) >= 8 or '1' in pad: + raise DecodeError(f'{len(pad)} trailing bits left, ones={pad.count("1")}') + if left != 1: + raise DecodeError(f'last chunk left={left}, expected 1') + # Chunk i covers m_i consecutive samples counted from sample 0. For n >= 1 (checked above) the + # loop reads exactly n - 1 tokens, which is sum(m) - left = sum(m) - 1 once left == 1, so no + # separate check that the counts sum to n is needed. A chunk's header sits in the stream + # immediately after its first sample, so quality is assigned by these cumulative ranges, not + # by header position. + quals = [q for m, q in chunks for _ in range(m)] + info = dict(h, counts=counts, pad_bits=len(pad), chunks=chunks) + return [_value(r, kind, signed) for r in vals], quals, info + + +def decode_block(b, t0=None): + """Bytes of one BinValues -> [(timestamp, value, quality)]. t0 defaults to the header's start.""" + vals, quals, info = decode_values(b) + start = t0 or info['t0'] + step = datetime.timedelta(milliseconds=info['period_ms']) + try: + return [(start + k * step, v, q) for k, (v, q) in enumerate(zip(vals, quals))] + except OverflowError: + raise DecodeError(f"sample times run past the last representable date from {start}") + + +def format_timestamp(dt): + """UTC to the millisecond, rounded: the header's start time is a double and carries microsecond noise.""" + try: + dt = dt + datetime.timedelta(microseconds=500) + except OverflowError: + raise DecodeError(f'timestamp {dt} is too late to format') + return f'{dt:%Y-%m-%dT%H:%M:%S}.{dt.microsecond // 1000:03d}Z' + + +def format_value(v): + return f'{v:.9g}' if isinstance(v, float) else str(v) # 9 digits round-trip a float32 + + +def block_from_line(line): + """The block bytes from a bare hex line or from a dump line of four `|`-separated fields.""" + fields = [f.strip() for f in line.split('|')] + if len(fields) == 1: + text = fields[0] + elif len(fields) == 4: + text = fields[3] + else: + raise DecodeError(f'expected hex or three columns then hex, got {len(fields)} fields') + try: + return bytes.fromhex(text) + except ValueError: + raise DecodeError(f'not hex: {text[:30]!r}') + + +def convert(lines, out, err): + """Write the CSV rows of every block in `lines`; return (blocks decoded, blocks failed).""" + writer = csv.writer(out, lineterminator='\n') + writer.writerow(['timestamp', 'value', 'quality']) + ok = bad = 0 + for number, line in enumerate(lines, 1): + if not line.strip(): + continue + try: + rows = [[format_timestamp(t), format_value(v), q] + for t, v, q in decode_block(block_from_line(line))] + except DecodeError as e: + bad += 1 + print(f'line {number}: {e}', file=err) + continue + ok += 1 + writer.writerows(rows) + return ok, bad + + +def main(argv=None): + parser = argparse.ArgumentParser(prog='wincc_binvalues.py', description=__doc__.split('\n\n')[0]) + parser.add_argument('dump', nargs='?', default='-', help='dump file, or - for standard input (default)') + try: + args = parser.parse_args(argv) + except SystemExit as e: + return e.code + try: + if args.dump != '-': + with open(args.dump, encoding='utf-8-sig') as fh: + lines = fh.read().splitlines() + else: + lines = sys.stdin.read().splitlines() + except (OSError, UnicodeDecodeError) as e: + print(f'cannot read input: {e}', file=sys.stderr) + return 2 + ok, bad = convert(lines, sys.stdout, sys.stderr) + if ok == 0 and bad == 0: + print('no blocks found in the input', file=sys.stderr) + return 1 if bad or ok == 0 else 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/skills/other-platforms/SKILL.md b/skills/other-platforms/SKILL.md index e34394d..cd6a48d 100644 --- a/skills/other-platforms/SKILL.md +++ b/skills/other-platforms/SKILL.md @@ -9,11 +9,13 @@ description: >- Designer (L5X/L5K, RLL neutral text XIC/XIO/OTE/TON, v36 IEC mnemonic renames, AOIs, FBD_TIMER in ST). Also reading a CODESYS/Machine Expert binary `.project` without the IDE (ST text, object inventory, build identity with `info`, comparing two versions with `diff`, - where a name is used with `xref`). Use when working with any of these tools or files. For - Siemens and Schneider use their dedicated skills. + where a name is used with `xref`). Also decoding the fast-archive blocks of a Siemens WinCC + Tag Logging segment (`TagCompressed.BinValues`) to CSV without WinCC. Use when working with + any of these tools or files. For Siemens PLCs (TIA Portal, S7) and Schneider use their + dedicated skills. --- -# Other platforms: Beckhoff, CODESYS, ABB, Rockwell +# Other platforms: Beckhoff, CODESYS, ABB, Rockwell, WinCC archives Load the reference for the platform in play. Each one lists the file format, the dialect's differences from IEC 61131-3, and the traps. @@ -26,6 +28,7 @@ differences from IEC 61131-3, and the traps. | ABB Freelance (DCS) | Freelance 2024 (announced 2024-10-02); manuals read: 2019 | read the `.csv` export; bulk-edit via BDM Excel; **never hand-edit `.prt`/`.csv` for re-import without a backup** | [references/abb.md](references/abb.md) | | ABB AC500 (Automation Builder) | CODESYS-based | PLCopenXML, `.export` (same-version only) | [references/abb.md](references/abb.md) | | Rockwell Studio 5000 | v38 (Sept 2025) | L5X component exports, RLL neutral text, L5K | [references/rockwell-logix.md](references/rockwell-logix.md) | +| Siemens WinCC 7 Tag Logging fast archive (SQL Server segment files, no WinCC) | derived on 7.5 SP2; other versions untested | **read only**: `TagCompressed.BinValues` blocks to CSV by script; undocumented format, the compressed archive type is not decoded, and the int32 signedness is a guess | [references/wincc-tag-logging-archive.md](references/wincc-tag-logging-archive.md) | ## Cross-platform rules @@ -60,6 +63,27 @@ checked on Machine Expert V2.x files only. - Details, what is decoded and what is a heuristic: [references/codesys-project-files.md](references/codesys-project-files.md). +## Reading a WinCC Tag Logging fast archive without WinCC + +Process history that lives in swapped-out segment files (`.mdf` / `.ldf`) of a WinCC 7 Tag +Logging fast archive is stored as compressed blocks in `TagCompressed.BinValues`. The format is +**undocumented**; the decoder is reverse-engineered on WinCC 7.5 SP2 and untested on other +versions. + +- Attach copies of the segment to a **separate** SQL Server instance (never a live WinCC), + dump the blocks of one tag, then + `python3 ${CLAUDE_PLUGIN_ROOT}/scripts/wincc_binvalues.py dump.txt > tag.csv` + (`timestamp,value,quality`; standard input works too). The query, the `sqlcmd -y` trap that + truncates long blocks, and the exit codes are in the reference. +- **Exclude quality other than 192 (`0xC0`) when averaging**; a bad chunk repeated the last good + value on an outage in the source data (observed, no automated check). +- A block that fails is reported by line and skipped. About 0.1 % of blocks failed in a large run, + essentially all on an unknown value code or on the first chunk header of a block written right + after a runtime restart (not decoded). A decoded value is not proof the archive was undamaged. +- **Not decoded**: archives of the compressed type, and the signedness of the 32-bit integer + code is a guess. Details and what is established: + [references/wincc-tag-logging-archive.md](references/wincc-tag-logging-archive.md). + Validate: `python3 ${CLAUDE_PLUGIN_ROOT}/scripts/plc_validate.py `. - TwinCAT rules: TC0xx. diff --git a/skills/other-platforms/references/wincc-tag-logging-archive.md b/skills/other-platforms/references/wincc-tag-logging-archive.md new file mode 100644 index 0000000..98a1690 --- /dev/null +++ b/skills/other-platforms/references/wincc-tag-logging-archive.md @@ -0,0 +1,218 @@ +# Reading WinCC Tag Logging fast-archive blocks offline + +For Siemens WinCC (classic, "WinCC 7") Tag Logging **fast archives** whose swapped-out segments +are SQL Server `.mdf` / `.ldf` files and where the values sit in `TagCompressed.BinValues` as a +compressed binary block. The plugin ships a decoder, `scripts/wincc_binvalues.py`, standard +library only, Python 3.9+. WinCC Unified, the WinCC OA product and the TIA Portal are different +products and are not covered here. + +## Status: undocumented and reverse-engineered + +**There is no Siemens documentation for this block format.** Everything below was worked out by +decoding real archives and checking the result against other data. Nothing is confirmed by +Siemens, and a newer or older WinCC may differ. Labels used on this page: + +- **Established**: a check that can fail was run and passed (stated next to each item). +- **Guess**: chosen because nothing seen contradicts it; the decoder pins it in a test, and a + value that would show the guess is wrong was never seen. +- **Not decoded**: the decoder does not try, or ignores the field. + +Derived on **WinCC 7.5 SP2** fast archives (two single-day segments for the format itself, then +669 segments for a bulk run); "7.5 SP2" is the maintainer's statement, not read from the segment +files. **Other WinCC versions are untested.** When the user's version +differs, or a block fails to decode, say so and treat the output as unverified. + +## Safe workflow + +1. **Never attach a segment to a live WinCC instance** (maintainer's instruction): use a separate + SQL Server instance, and work on **copies** of the `.mdf` / `.ldf` pair. Microsoft documents + that attaching upgrades a database created by an older SQL Server version, and that it cannot + then be attached to an earlier version again + ([Database detach and attach](https://learn.microsoft.com/sql/relational-databases/databases/database-detach-and-attach-sql-server)). + The work behind this page used SQL Server 2019; the lowest version that can attach WinCC 7.5 + segments was not checked. +2. Attach the pair. The segment files whose names contain `TLG_F` held the fast archive here. + + ```sql + CREATE DATABASE [seg1] ON (FILENAME='/seg1.mdf'), (FILENAME='/seg1.ldf') FOR ATTACH; + ``` + +3. Dump the blocks of one tag, in the line shape the decoder reads (three columns it ignores, + then the hex). The table and column names `TagCompressed`, `Archive`, `ValueID`, `ValueName`, + `Timebegin` and `BinValues` are **established** (used in the bulk run); `ValueName` held + `\`, for example `Archive\Tag_1`. + + ```sql + SET NOCOUNT ON; + SELECT CONVERT(varchar(23), t.Timebegin, 121), t.ValueID, DATALENGTH(t.BinValues), + CONVERT(varchar(max), t.BinValues, 2) + FROM TagCompressed t JOIN Archive a ON a.ValueID = t.ValueID + WHERE a.ValueName = N'Archive\Tag_1' ORDER BY t.Timebegin; + ``` + + ```text + sqlcmd -S -d seg1 -s '|' -y 0 -Q "" > dump.txt + ``` + + This is for a POSIX shell; the `'|'` quoting is shell-specific, so quote the `|` as your shell + requires. **The dump must be UTF-8** (the decoder opens a file as UTF-8 and accepts a byte order + mark). A UTF-16 file is refused with exit 2 and is not sniffed or converted: convert it first. + Which `sqlcmd` and shell combinations write which encoding was **not checked** here. + + **`-y` matters.** `sqlcmd` cuts a `varchar(max)` column to `-y` characters (default 256; + [sqlcmd utility](https://learn.microsoft.com/sql/tools/sqlcmd/sqlcmd-utility)), and a byte is + two hex characters, while the block length field is 16 bits (up to 65 535 bytes). `-y 0` is + unlimited (Microsoft warns it can be slow on large results) and prints no header line, so it + takes no `-h -1`: `sqlcmd` refuses the pair ("The -h and the -y 0 options are mutually + exclusive"). **Do not add `-w`.** Checked on mssql-tools18 with an 80 000-character column: + `-y 0` alone wrote one physical line, and `-w 65535` (the largest width) split it into two. A + split block reads as failed blocks, because the decoder takes one physical line as one block. + Other `sqlcmd` builds were not checked: confirm that the dump has one line per block. The bulk + run used `-h -1 -y 8000`, which cuts a block at 4000 bytes. A cut block fails the checksum or the length + check. The failing blocks of the bulk run were classified later (see Bulk run) and fell into + other classes, so a cut block did not explain them; the checksum and length checks stay the + guard against a cut. +4. Decode: `python3 ${CLAUDE_PLUGIN_ROOT}/scripts/wincc_binvalues.py dump.txt > tag.csv`, or one + block as hex on standard input. The CSV columns are `timestamp,value,quality`: UTC to the + millisecond, the value, the OPC quality byte in decimal (192 = `0xC0` good, 8 = `0x08` bad). + Exit 0 clean, 1 if some blocks failed (their line numbers go to standard error, the others are + still written) or none were found, 2 for unreadable input or a usage error (such as too many + arguments). +5. **Exclude `quality != 192` when averaging.** **Established** (observed on an outage in the + source data; no automated check): a bad-quality chunk carries the last good value frozen, not + a gap. +6. Never commit real archive bytes, tag names or segment names to a public repository. + +## Block header (37 bytes) + +| Offset | Size | Meaning | Status | +|---|---|---|---| +| 0 | 1 | XOR of bytes 1 to the end (checksum) | **Established**: matched on all 3179 blocks examined | +| 1-3 | 3 | `05 04 00`, constant | constant; meaning guessed (byte 1 = 5 might be a compression mode) | +| 4 | 1 | value code, the representation of every value in the block (table below) | **Established** | +| 5-6 | 2 | `20 20`, constant | not decoded | +| 7-8 | 2 | uint16 little-endian total block length in bytes | **Established** | +| 9-12 | 4 | uint32 LE sample count `n` (the span in periods plus one) | **Established** | +| 13-20 | 8 | double, OLE automation date of the first sample (epoch 1899-12-30). It equals the segment table's `Timebegin`, which WinCC stores in UTC (vendor behaviour, not re-verified here), so the CSV is UTC | **Established** | +| 21-24 | 4 | uint32 LE period in ms (1000 in the two-day totals below; 500 was seen on a few binary-flag tags outside those totals). A period of 0 is rejected | **Established** | +| 25-36 | 12 | a double (a second timestamp) and a uint32 | **Not decoded**, ignored: a bad value there never fails a block | + +## Value codes (header byte 4) + +| Code | Value | Status | +|---|---|---| +| 3 | float32, little-endian | width **established** | +| 4 | int16 | width **established**; signed because negative values occurred | +| 5 | int8 | width **established**; signed because negative values occurred | +| 6 | uint8 | width **established**; unsigned (values above 127 occurred) | +| 7 | uint16 | width **established**; unsigned (a Wh counter matched a float kWh counter times 1000 on samples above 32 767) | +| 8 | int32 | width **established**; **signedness is a guess**: no value of 2^31 or more occurred | + +Widths follow from bit accounting: the whole stream of every block is consumed exactly. A float +tag switches to code 4, 5, 6 or 8 for a block whose values are all whole numbers, so a block ends +when the value stops being integral. In the two days the format was derived on no other code was +seen; the bulk run later met code `1` (below). A float64 code presumably exists (a guess). + +## Bit stream (from byte 37, read MSB-first) + +A multi-byte value is its little-endian bytes in order, each byte MSB-first. **Established** by +decoding every block examined without a bit left over. + +1. The initial value: a `0` bit, then the value (`w` bits). +2. A chunk header (84 bits, below). +3. One token per later sample: + - `11`: the same value as the previous sample; + - `0` then `w` bits: a new value; + - `10` then `cc`: a value from the history (below). `cc` = `01`, `10`, `11` gives index 1, 2, + 3; `cc` = `00` is followed by 4 more bits `x` and gives index 4 + `x`. +4. Zero bits up to the byte boundary (fewer than 8). + +### Chunk header (84 bits) + +| Bits | Content | +|---|---| +| 10 | `m`, the samples in the chunk (at most 1023) | +| 8 | zeros | +| 8 | `q`, the OPC quality byte (`0xC0` good, `0x08` bad / not connected seen) | +| 16 | `0001000000000000`, **whose last bit is a daylight-saving flag** | +| 10 | `m` again | +| 24 | zeros | +| 8 | `q` again | + +- A chunk is cut at 1023 samples or where the quality changes (**established**: every header's + two counts and two quality bytes agreed). +- **Daylight-saving flag** (a late finding): across 669 segments the last bit of the 16-bit + constant was 1 in every summer-time segment and 0 in every winter-time one. That is a + correlation; Siemens does not document the bit. The decoder accepts both values and ignores it. +- **Bookkeeping**: the `m` of all chunks add up to `n`. Chunk *i* covers the samples from the sum of + the earlier counts, counted from the initial value; its header sits immediately after its first + sample, so quality is assigned by these ranges, not by where the header sits. A block whose + last chunk filled exactly ends with an empty header (`m` = 1, no tokens). + +### Value history (move-to-front, 19 entries) + +A reference token reads from a list of the 19 most recent distinct values. Index 0 is the current +value; a new or referenced value moves to the front; the 19th is pushed out. **Established** by a +check that can fail: an encoder never spends a full value on one it could reference, so counting +the new-value tokens whose value was already in the decoder's history must give zero. With a +19-entry move-to-front list it did, on two days (0 of 382 021 and 0 of 292 581 tokens); a +FIFO of all changes gave 332 and 164, and a list of 20 gave 918 and 361. Index 19 (`x` = 15) never +occurs, and the decoder rejects it. + +**Guess: the history compares raw bits.** Entries are kept and matched by their raw value bytes, +so `0.0` and `-0.0` are two entries and a NaN matches an identical NaN. WinCC's own rule is +**unverified**; a test pins the choice with a hand-written bit vector. + +### Timestamps + +There are no time tokens: sample *k* is at the header's first time plus *k* times the period +(**established**: for the tags checked, the last sample matched the block's end time and the +concatenated blocks had no gap). + +## Evidence + +- **Format**: all 3179 fast-archive blocks of two 24-hour segments (81 and 30 tags, 6 998 400 and + 2 592 000 samples) decode with the checksum matching, every chunk header consistent, exactly `n` + samples, and fewer than 8 zero padding bits left. +- **Values**: the integral of an inverter's power tag matched the rise of its two energy counters + to within 0.2 % (ratios 0.998 to 1.000) over four windows. The maintainer's check; the scripts + and data are not in this repository. +- **Corruption control**: one flipped bit is caught by the checksum. With the checksum repaired, a + flip inside a value field decodes without any error: **the stream checks cannot catch every + corruption**, so a decoded value is not proof the archive was undamaged. +- **Bulk run**: 669 segments, 646 decode without a single failing block. The other 23 had a few + failing blocks (about 540 of about 450 000, 0.1 %). The failing blocks were classified afterwards + (see below): an unknown value code, and the first chunk header of a block written right after a + runtime restart. Together these account for essentially all of the failing blocks seen. Those + two classes are not decoded. + +## What is not decoded or not established + +- **Archives of the compressed type** (a second archive type in the same segments: header byte 4 + = 0, a 60 000 ms period, and a different chunk-header constant, `0001000010000000`): **not + decoded**. The decoder rejects them rather than guess. +- **Value code `1`**: occurs in the bulk run, is not in the code table, and its width is + unknown; the decoder rejects the block as an unknown value code. +- **Blocks written immediately after a runtime restart**: they fail the chunk-header check + ("count mismatch" or "unexpected chunk header" at the first header). They have a different + first-chunk layout, which is not decoded. +- Header bytes 1-3, 5-6 and 25-36: **not checked**, so a damaged value there does not fail a + block. Nor are the value of a quality byte or the daylight-saving bit checked. +- **Signedness of code 8** (a guess), and codes other than 3 to 8. +- Qualities other than `0xC0` and `0x08` never occurred; any 8-bit value is passed through. +- Only two days were examined to derive the format; other years may hold codes or tokens not seen. + The decoder raises `DecodeError` for a block that fails one of its checks (checksum, length, + sample count and period, value code, chunk headers, stream consistency) instead of guessing; it + accepts anything those checks and the unchecked fields above let through. +- The meaning of the daylight-saving bit beyond the correlation above. +- WinCC versions other than 7.5 SP2. + +## Tests + +`tests/test_wincc_binvalues.py` builds **synthetic** blocks with a small encoder written from this +page: each value code, a chunk split at 1023 samples, a quality change, short and long history +references, the daylight-saving bit set and clear, and the rejected inputs (checksum, length, +value code, chunk header, padding, truncation). The encoder is the same author's reading of the +format, so these tests show that the decoder and the page agree, not that the page matches +WinCC. One test, for the history rule, uses a bit vector written out by hand without the encoder; +it is still the same reading of the format. No real block, tag name or segment is committed. diff --git a/skills/siemens/SKILL.md b/skills/siemens/SKILL.md index 91c3d4e..8e716bb 100644 --- a/skills/siemens/SKILL.md +++ b/skills/siemens/SKILL.md @@ -105,3 +105,8 @@ Run `python3 ${CLAUDE_PLUGIN_ROOT}/scripts/plc_validate.py `. | `.s7dcl` | SD0xx: `NETWORK`/`RUNG` nesting only | None of these is a TIA compile. Tell the user to compile in TIA and to fix what it reports. + +WinCC (classic, 7.x) Tag Logging archive segments are not TIA files. To read the fast-archive +blocks of one offline, see +[../other-platforms/references/wincc-tag-logging-archive.md](../other-platforms/references/wincc-tag-logging-archive.md): +undocumented format, reverse-engineered on WinCC 7.5 SP2. diff --git a/tests/test_wincc_binvalues.py b/tests/test_wincc_binvalues.py new file mode 100644 index 0000000..7df9a06 --- /dev/null +++ b/tests/test_wincc_binvalues.py @@ -0,0 +1,551 @@ +"""WinCC Tag Logging fast-archive decoder, tested on SYNTHETIC blocks only. + +The encoder below is written from the format notes in +skills/other-platforms/references/wincc-tag-logging-archive.md. It has its own value table and bit +packing, but it is still the same author's reading of the format, and it shares the decoder's idea +of the stream layout: a round trip proves the decoder and this encoder agree and that every guard +fires, not that the format is understood correctly. Only `test_history_compares_raw_bits` is +written out by hand, without the encoder; it is still the same reading of the format. Whether the +format is understood correctly is checked on real archives, which are never committed here. +""" + +from __future__ import annotations + +import contextlib +import datetime +import io +import math +import struct +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +HERE = Path(__file__).resolve().parent +SCRIPT = HERE.parent / "scripts" / "wincc_binvalues.py" +sys.path.insert(0, str(HERE.parent / "scripts")) + +import wincc_binvalues as W # noqa: E402 +from wincc_binvalues import DecodeError # noqa: E402 + +GOOD, BAD = 0xC0, 0x08 +OLE_EPOCH = datetime.datetime(1899, 12, 30) +T0 = datetime.datetime(2024, 1, 1, 6, 0, 0) # a fraction of a day that is exact in a double +MAX_CHUNK = 1023 +HIST = 19 +# value code -> struct format of one value (independent of the decoder's table) +FORMATS = {3: " str: + return "".join(f"{x:08b}" for x in data) + + +def value_bits(code, v): + return bits_of(struct.pack(FORMATS[code], v)) + + +def chunk_header(m, q, dst=False, m2=None, q2=None, lead="0" * 8, const=None, tail="0" * 24): + """84 bits: m(10) lead(8) q(8) const(16, last bit = DST) m(10) tail(24) q(8).""" + const = const if const is not None else CHUNK_CONST[:-1] + ("1" if dst else "0") + return (f"{m:010b}" + lead + f"{q:08b}" + const + f"{m if m2 is None else m2:010b}" + tail + + f"{q if q2 is None else q2:08b}") + + +NEW = "0" +SAME = "11" + + +def ref_token(index): + if 1 <= index <= 3: + return "10" + f"{index:02b}" + return "10" + "00" + f"{index - 4:04b}" + + +def split_chunks(quals): + """Chunks are cut at a quality change and at 1023 samples.""" + chunks, run_q, run = [], None, 0 + for q in quals: + if q == run_q and run < MAX_CHUNK: + run += 1 + else: + if run: + chunks.append((run, run_q)) + run_q, run = q, 1 + chunks.append((run, run_q)) + return chunks + + +def encode_stream(code, values, quals, dst=False): + """Initial value, then one token per later sample; chunk j's header follows S_(j-1) tokens.""" + chunks = split_chunks(quals) + starts = [0] + for m, _ in chunks: + starts.append(starts[-1] + m) + out, hist, nxt = [NEW + value_bits(code, values[0])], [value_bits(code, values[0])], 0 + + def headers(done): + nonlocal nxt + if nxt < len(chunks) and starts[nxt] == done: + out.append(chunk_header(chunks[nxt][0], chunks[nxt][1], dst)) + nxt += 1 + + headers(0) + for k in range(1, len(values)): + key = value_bits(code, values[k]) # the history compares raw value bits, not Python equality + if key == hist[0]: + out.append(SAME) + elif key in hist: + out.append(ref_token(hist.index(key))) + hist.remove(key) + hist.insert(0, key) + else: + out.append(NEW + key) + hist.insert(0, key) + del hist[HIST:] + headers(k) + return "".join(out) + + +def assemble(code, n, stream_bits, t0=T0, period_ms=1000, length=None, t0_days=None): + """37-byte header plus the bit stream (zero padded to a byte), with a correct XOR checksum.""" + body = bytes(int((stream_bits + "0" * (-len(stream_bits) % 8))[i:i + 8], 2) + for i in range(0, len(stream_bits) + (-len(stream_bits) % 8), 8)) + days = t0_days if t0_days is not None else (t0 - OLE_EPOCH).total_seconds() / 86400 + head = bytearray(37) + head[1:4] = b"\x05\x04\x00" + head[4] = code + head[5:7] = b"\x20\x20" + head[7:9] = (length if length is not None else 37 + len(body)).to_bytes(2, "little") + head[9:13] = n.to_bytes(4, "little") + head[13:21] = struct.pack("= 2**31 was ever seen); this pins the guess. + self.check(8, [-100000, 70000, 2 ** 31 - 1, -(2 ** 31)]) + + def test_a_single_sample(self): + self.check(3, [2.5]) + + def test_constant_block_uses_only_same_tokens(self): + info = self.check(4, [5] * 50) + self.assertEqual((info["counts"]["same"], info["counts"]["new"]), (49, 1)) + + def test_every_new_value_costs_a_full_value(self): + info = self.check(4, [1, 2, 3, 4, 5, 6]) + self.assertEqual((info["counts"]["new"], info["counts"]["ref"]), (6, 0)) + + def test_header_fields(self): + info = W.decode_values(encode_block([1.0, 2.0, 3.0], period_ms=500))[2] + self.assertEqual((info["n"], info["period_ms"], info["code"], info["t0"]), (3, 500, 3, T0)) + + +class Header(unittest.TestCase): + def test_sample_count_and_period_are_four_bytes_wide(self): + head = bytearray(37) + head[9:13] = (0x00010000).to_bytes(4, "little") + head[21:25] = (0x00010000).to_bytes(4, "little") + h = W.parse_header(bytes(head)) + self.assertEqual((h["n"], h["period_ms"]), (65536, 65536)) + + +class Chunks(unittest.TestCase): + def test_quality_per_sample_follows_the_chunk_ranges(self): + quals = [GOOD, GOOD, BAD, BAD, GOOD] + got = W.decode_values(encode_block([1, 2, 3, 4, 5], code=4, quals=quals))[1] + self.assertEqual(got, quals) + + def test_quality_change_cuts_a_chunk(self): + quals = [GOOD, GOOD, BAD, BAD, GOOD] + info = W.decode_values(encode_block([1, 2, 3, 4, 5], code=4, quals=quals))[2] + self.assertEqual(info["chunks"], [(2, GOOD), (2, BAD), (1, GOOD)]) + + def test_chunk_is_cut_at_1023_samples(self): + values = [float(i % 7) for i in range(1030)] + vals, quals, info = W.decode_values(encode_block(values)) + self.assertEqual((info["chunks"], vals), ([(1023, GOOD), (7, GOOD)], values)) + + def test_a_block_that_fills_its_last_chunk_exactly_ends_with_an_empty_header(self): + values = [float(i % 5) for i in range(1024)] + vals, quals, info = W.decode_values(encode_block(values)) + self.assertEqual((info["chunks"], vals), ([(1023, GOOD), (1, GOOD)], values)) + + def test_timestamps_step_by_the_period(self): + rows = W.decode_block(encode_block([1.0, 2.0, 3.0], period_ms=500)) + self.assertEqual([r[0] for r in rows], [T0 + datetime.timedelta(milliseconds=500 * k) for k in range(3)]) + + def test_start_time_can_be_overridden(self): + later = datetime.datetime(2030, 5, 6, 7, 8, 9) + self.assertEqual(W.decode_block(encode_block([1.0, 2.0]), t0=later)[0][0], later) + + +class History(unittest.TestCase): + def test_short_index_references_a_recent_value(self): + vals, quals, info = W.decode_values(encode_block([1, 2, 1], code=4)) + self.assertEqual((vals, info["counts"]["ref"]), ([1, 2, 1], 1)) + + def test_each_short_index(self): + # after 1,2,3,4 the history is [4,3,2,1]: referencing 3, 2 and 1 uses indexes 1, 2 and 3 + # (each reference moves its value to the front, so re-reading the list is part of the test) + values = [1, 2, 3, 4, 3, 3, 2, 1, 4] + vals, quals, info = W.decode_values(encode_block(values, code=4)) + self.assertEqual((vals, info["counts"]["ref"]), (values, 4)) + + def test_long_index_references_an_older_value(self): + values = [1, 2, 3, 4, 5, 6, 1] # history [6..1]: the 1 sits at index 5 = 4 + 1 + vals, quals, info = W.decode_values(encode_block(values, code=4)) + self.assertEqual((vals, info["counts"]["ref"]), (values, 1)) + + def test_oldest_entry_is_index_18(self): + values = list(range(1, 20)) + [1] # 19 distinct values, then the first one again + vals, quals, info = W.decode_values(encode_block(values, code=4)) + self.assertEqual((vals, info["counts"]["ref"]), (values, 1)) + + def test_history_compares_raw_bits(self): + # Written out by hand, not by the encoder. float32 0.0 is four zero bytes, -0.0 is + # 00 00 00 80 (stream order). Python says 0.0 == -0.0; by raw bits they are two entries, so + # after NEW 0.0 and NEW -0.0 index 1 is the 0.0 again. + stream = ( + "0" + "00000000" * 4 # initial value: 0.0 + + "0000000011" + "00000000" + "11000000" # chunk header: m = 3, lead 0, q = 0xC0 + + "0001000000000000" # constant, DST bit clear + + "0000000011" + "0" * 24 + "11000000" # m again, tail, q again + + "0" + "00000000" * 3 + "10000000" # NEW: -0.0 + + "10" + "01" # reference, index 1 + ) + vals = W.decode_values(assemble(3, 3, stream))[0] + self.assertEqual([math.copysign(1, v) for v in vals], [1.0, -1.0, 1.0]) + self.assertEqual(vals, [0.0, 0.0, 0.0]) + + def test_index_19_is_rejected(self): + # 20 distinct values; the 20th pushes the first out, so a reference to index 19 has no entry + stream = NEW + value_bits(4, 1) + chunk_header(21, GOOD) + stream += "".join(NEW + value_bits(4, v) for v in range(2, 21)) + "10" + "00" + "1111" + with self.assertRaisesRegex(DecodeError, "history index 19"): + W.decode_values(assemble(4, 21, stream)) + + def test_index_beyond_the_history_is_rejected(self): + stream = NEW + value_bits(4, 1) + chunk_header(2, GOOD) + ref_token(1) + with self.assertRaisesRegex(DecodeError, "history index 1 beyond 1"): + W.decode_values(assemble(4, 2, stream)) + + +class DaylightSavingFlag(unittest.TestCase): + VALUES = [1, 2, 2, 3] + + def test_flag_set_decodes(self): + self.assertEqual(W.decode_values(encode_block(self.VALUES, code=4, dst=True))[0], self.VALUES) + + def test_flag_clear_decodes(self): + self.assertEqual(W.decode_values(encode_block(self.VALUES, code=4, dst=False))[0], self.VALUES) + + def test_the_two_blocks_really_differ(self): + self.assertNotEqual(encode_block(self.VALUES, code=4, dst=True), encode_block(self.VALUES, code=4, dst=False)) + + +class Rejects(unittest.TestCase): + GOOD_BLOCK = encode_block([1.0, 2.0, 3.0, 2.0]) + + def decode(self, raw): + return W.decode_values(raw) + + def test_the_control_block_decodes(self): + self.assertEqual(self.decode(self.GOOD_BLOCK)[0], [1.0, 2.0, 3.0, 2.0]) + + def test_input_shorter_than_the_header(self): + for raw in (b"", self.GOOD_BLOCK[:36]): + with self.subTest(length=len(raw)), \ + self.assertRaisesRegex(DecodeError, "shorter than the 37-byte header"): + self.decode(raw) + + def test_bad_checksum(self): + raw = bytes([self.GOOD_BLOCK[0] ^ 0x01]) + self.GOOD_BLOCK[1:] + with self.assertRaisesRegex(DecodeError, "checksum"): + self.decode(raw) + + def test_bad_length_field(self): + raw = encode_block([1.0, 2.0], length=999) + with self.assertRaisesRegex(DecodeError, "length field 999"): + self.decode(raw) + + def test_unknown_value_code(self): + with self.assertRaisesRegex(DecodeError, "unknown value code 9"): + self.decode(with_byte(self.GOOD_BLOCK, 4, 9)) + + def test_unrepresentable_start_time(self): + raw = encode_block([1.0, 2.0], t0_days=1e300) + with self.assertRaisesRegex(DecodeError, "start time"): + self.decode(raw) + + def test_start_time_that_is_not_a_number(self): + with self.assertRaisesRegex(DecodeError, "start time"): + self.decode(encode_block([1.0, 2.0], t0_days=float("nan"))) + + def test_a_block_with_zero_samples_is_rejected_even_with_one_sample_in_the_stream(self): + raw = assemble(3, 0, encode_stream(3, [1.0], [GOOD])) + with self.assertRaisesRegex(DecodeError, "sample count 0"): + self.decode(raw) + + def test_a_period_of_zero_is_rejected(self): + with self.assertRaisesRegex(DecodeError, "period of 0"): + self.decode(encode_block([1.0, 2.0], period_ms=0)) + + def test_garbage_in_the_unknown_header_bytes_does_not_fail_a_block(self): + # bytes 25-36 are not understood and not needed: a NaN there must not cost the block + raw = seal(self.GOOD_BLOCK[:25] + struct.pack("