diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fa31d998..a745f407 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -191,6 +191,92 @@ jobs: run: | docker run --rm --volume $(pwd):/note-c/ --workdir /note-c/ --entrypoint ./scripts/run_unit_tests.sh ghcr.io/blues/note_c_ci:latest --mem-check --low-mem --single-precision + run_layout_matrix_unit_tests: + # The historical J layout is the default; NOTE_C_STORAGE_OPTIMIZATION packs + # keys and short string values into the node's own allocation. Both layouts + # are shipped, so both must stay green. + # + # Precision is crossed in because it is not independent of layout: JNUMBER + # is a float under NOTE_C_SINGLE_PRECISION, which changes the size and tail + # padding of the inline region the packed layout carves its storage out of. + # A defect confined to one cell of this matrix is otherwise easy to miss. + # + # Low memory is crossed in as an independent axis too, because it changes + # which code compiles at all. + # + # Between this matrix and the two standalone jobs, all eight cells of + # {layout} x {precision} x {low-mem} are built and run: default/double is + # run_unit_tests, default/low-mem/single is run_low_mem_unit_tests, and the + # remaining six are below. + runs-on: ubuntu-latest + if: ${{ always() }} + needs: [build_ci_docker_image] + + strategy: + fail-fast: false + matrix: + include: + - name: default layout, single precision + flags: --single-precision + - name: storage optimization, default precision + flags: --storage-optimization + - name: storage optimization, single precision + flags: --storage-optimization --single-precision + - name: storage optimization, low memory, single precision + flags: --storage-optimization --low-mem --single-precision + - name: default layout, low memory, double precision + flags: --low-mem + - name: storage optimization, low memory, double precision + flags: --storage-optimization --low-mem + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Load CI Docker image + # Only load the Docker image artifact if build_ci_docker_image actually + # ran (e.g. it wasn't skipped and was successful). + if: ${{ needs.build_ci_docker_image.result == 'success' }} + uses: ./.github/actions/load-ci-image + + - name: Run tests (${{ matrix.name }}) + run: | + docker run --rm --volume $(pwd):/note-c/ --workdir /note-c/ --entrypoint ./scripts/run_unit_tests.sh ghcr.io/blues/note_c_ci:latest --mem-check ${{ matrix.flags }} + + run_j_layout_ab_comparison: + # Proves the two layouts render byte-identically, and publishes the memory + # difference in the job log. + # + # Run at both precisions. JNUMBER is a float under NOTE_C_SINGLE_PRECISION, + # which removes the historical layout's tail padding on its own: sizeof(J) + # is 40 in BOTH layouts there, so the saving is materially smaller and comes + # entirely from inline packing. Publishing only the double-precision table + # would overstate the win for single-precision targets. + runs-on: ubuntu-latest + if: ${{ always() }} + needs: [build_ci_docker_image] + + strategy: + fail-fast: false + matrix: + include: + - name: double precision + flags: "" + - name: single precision + flags: --single-precision + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Load CI Docker image + if: ${{ needs.build_ci_docker_image.result == 'success' }} + uses: ./.github/actions/load-ci-image + + - name: Compare default and storage-optimized J layouts (${{ matrix.name }}) + run: | + docker run --rm --volume $(pwd):/note-c/ --workdir /note-c/ --entrypoint ./scripts/run_ab_layout_comparison.sh ghcr.io/blues/note_c_ci:latest ${{ matrix.flags }} + run_astyle: runs-on: ubuntu-latest if: ${{ always() }} diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 9be6e6f4..0b95bc1e 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -35,7 +35,7 @@ Use this file to understand what the architecture is meant to be. Use `source-re - `n_serial.c`: serial transport implementation and chunked newline-framed serial transmit/receive behavior. - `n_i2c.c`: I2C transport implementation and chunked newline-framed I2C transmit/receive behavior. - `n_hooks.c`: global function-pointer hook registry, active-interface dispatch, and invocation of platform hooks for memory, time, mutexes, debug output, and transports. -- `n_cjson.c`, `n_cjson.h`, `n_cjson_helpers.c`: bundled JSON representation and helper APIs. +- `n_cjson.c`, `n_cjson.h`, `n_cjson_helpers.c`: bundled JSON representation and helper APIs. The `J` node has two compile-time storage layouts; see "JSON node storage" below. - `n_helpers.c`, `n_str.c`, `n_printf.c`, `n_atof.c`, `n_ftoa.c`, `n_b64.c`, `n_cobs.c`, `n_md5.c`, `n_const.c`, `n_ua.c`: portability helpers, encoding, formatting, constants, and utility behavior. - `test/`: unit tests and mocks for protecting SDK behavior without requiring real hardware. - `scripts/`: local automation for checks, documentation, and release support. @@ -68,6 +68,17 @@ Applications normally build requests as `J` objects, send them through `NoteRequ Transport and platform behavior is supplied through hooks so the same core code can run on microcontrollers, embedded Linux, tests, and other C/C++ environments. Serial and I2C transports move raw newline-framed bytes through hook dispatch. Binary payload helpers, not the transport implementations, own COBS framing and MD5 verification. +## JSON Node Storage + +`J` has two storage layouts, selected at compile time. Both produce byte-identical JSON and expose the same API; they differ only in how a node's key and string value are allocated. + +- **Historical layout (default).** A node is 48 bytes on a 32-bit target. A member's key and its string value each occupy their own heap allocation, so `"key":"value"` costs three allocations. +- **`NOTE_C_STORAGE_OPTIMIZATION`.** A node is 40 bytes, and a member's key and short string value are carved out of the node's own allocation, so the same member costs one allocation. The bytes occupied by `valueint` and `valuenumber` — which a `JString`/`JRaw` node never uses — hold that content, and a `objlen` member records the node's allocation size. Over a representative corpus this removes roughly 38% of allocations and 26% of the heap a parsed document holds. + +Ownership is expressed by four `type` flags: `JIsReference` and `JStringIsConst` (pre-existing, unchanged meanings), plus `JValueInline` and `JKeyInline`, which are only set under the optimization and record that a pointer addresses the node's own allocation. Every mutation of `valuestring` and `string` routes through storage helpers in `n_cjson.c`, so the layout-conditional code stays confined to those helpers; the parser, printer, and public API are layout-agnostic. + +The optimization is **off by default**, deferred pending a planned external beta of the historical layout, and because it changes the public representation of `J`: `sizeof(J)` changes, member offsets move, `type` narrows from `int` to `uint16_t`, `objlen` is added, and on a string node `valueint`/`valuenumber` may hold character data. note-c is always compiled from source as part of the customer solution, so the requirement is that every translation unit seeing `J` uses the same setting — the note-c sources and the customer sources alike. The CMake option propagates `PUBLIC` to enforce that, exactly as `NOTE_C_SINGLE_PRECISION` does. See `docs/architecture/decisions/0002-j-node-storage-layout.md`. + ## Public Contracts The main compatibility contracts are: @@ -77,7 +88,8 @@ The main compatibility contracts are: - Request/response ownership semantics for public transaction APIs. - Hook signatures for serial, I2C, memory, mutex, time, and debug output. - Notecard request/response semantics and timeout/retry behavior. -- Build configuration behavior for low-memory, single-precision, user-agent, CRC, and portability-helper variants. +- Build configuration behavior for low-memory, single-precision, storage-optimization, user-agent, CRC, and portability-helper variants. +- The in-memory representation of `J` itself, which downstream code reads directly. `NOTE_C_STORAGE_OPTIMIZATION` changes that representation and is therefore opt-in; see "JSON node storage". - Version constants and release expectations documented in `README.md`. Breaking changes to these contracts require deliberate versioning, migration notes, and architecture documentation updates. @@ -88,6 +100,8 @@ Breaking changes to these contracts require deliberate versioning, migration not Build configuration is part of the portability model. CMake detects platform `strlcpy`/`strlcat` support and only includes bundled `n_str.c` helpers when needed. Low-memory builds disable user-agent support and request CRC paths, omit `n_ua.c`, use compact error/log constants, and reduce allocation chunk size. +Two CMake options change the public representation rather than only behavior, and are propagated `PUBLIC` so every translation unit that sees `J` agrees: `NOTE_C_SINGLE_PRECISION` (which changes the width of `JNUMBER`) and `NOTE_C_STORAGE_OPTIMIZATION` (which changes the layout of `J`). note-c is always compiled from source as part of the consuming solution, so this is a single-build consistency requirement. + ## Runtime Model At runtime, host code initializes the relevant hooks, constructs Notecard requests, and calls `note-c` APIs. `note-c` serializes requests, sends bytes through the selected hook-backed transport, parses responses, and returns JSON objects or status to the caller. diff --git a/CMakeLists.txt b/CMakeLists.txt index 2803711f..5877464c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -24,6 +24,13 @@ option(NOTE_NODEBUG "Build the library without debug information." OFF) option(NOTE_C_NO_LIBC "Audit libc dependencies by linking the note_c test/audit library without libc, generating errors for any undefined symbols." OFF) option(NOTE_C_SHOW_MALLOC "Build the library with flags required to log memory usage." OFF) option(NOTE_C_SINGLE_PRECISION "Use single precision for JSON floating point numbers." OFF) +# Opt in to the packed J layout, which carves a member's key and short string +# value out of the node's own allocation instead of giving each its own heap +# block. The historical layout is the default. Like NOTE_C_SINGLE_PRECISION this +# changes sizeof(J), so it is propagated PUBLIC and must match between the +# library and every consumer. See docs/architecture/decisions/ +# 0002-j-node-storage-layout.md. +option(NOTE_C_STORAGE_OPTIMIZATION "Pack J keys and short string values into the node's own allocation (smaller, fewer allocations; changes the public J representation)." OFF) option(NOTE_C_HEARTBEAT_CALLBACK "Enable heartbeat callback support." OFF) # NOTE_C_NO_LIBC is a link-time undefined-symbol audit (see @@ -103,6 +110,9 @@ function(note_c_configure_common target) if(NOTE_C_SINGLE_PRECISION) target_compile_definitions(${target} PUBLIC NOTE_C_SINGLE_PRECISION) endif() + if(NOTE_C_STORAGE_OPTIMIZATION) + target_compile_definitions(${target} PUBLIC NOTE_C_STORAGE_OPTIMIZATION) + endif() if(NOTE_C_HEARTBEAT_CALLBACK) target_compile_definitions(${target} PUBLIC NOTE_C_HEARTBEAT_CALLBACK) endif() diff --git a/docs/architecture/architecture.html b/docs/architecture/architecture.html index 8c7e1dcd..f55305ef 100644 --- a/docs/architecture/architecture.html +++ b/docs/architecture/architecture.html @@ -928,7 +928,7 @@

Nodes

- +