From 8d26aafc601855ad982f27edb7e5651eb119c0a5 Mon Sep 17 00:00:00 2001 From: Rastislav Date: Mon, 28 Sep 2026 14:08:54 +0200 Subject: [PATCH 1/2] init --- .editorconfig | 11 + .github/ISSUE_TEMPLATE/bug_report.yml | 23 ++ .github/ISSUE_TEMPLATE/config.yml | 1 + .github/ISSUE_TEMPLATE/feature_request.yml | 13 + .github/pull_request_template.md | 7 + .github/workflows/ci.yml | 78 +++++ .github/workflows/conan-center.yml | 59 ++++ .github/workflows/release.yml | 70 +++++ .gitignore | 51 +++ CMakeLists.txt | 50 +++ LICENSE | 19 ++ Makefile | 12 + README.md | 159 +++++++++- SECURITY.md | 5 + cmake/txms.pc.in | 8 + cmake/txmsConfig.cmake.in | 3 + conanfile.py | 66 ++++ include/txms.h | 33 ++ packaging/LICENSE | 21 ++ packaging/conan-center/conanfile.py | 62 ++++ src/internal.h | 10 + src/txms.c | 52 +++ src/txms_decode.c | 33 ++ src/txms_encode.c | 50 +++ src/txms_hex.c | 30 ++ src/txms_utf16.c | 67 ++++ src/unicode_ranges.h | 348 +++++++++++++++++++++ test_package/CMakeLists.txt | 5 + test_package/conanfile.py | 26 ++ test_package/test_package.c | 15 + tests/VALIDATION.md | 30 ++ tests/compatibility.mjs | 26 ++ tests/corpus/escaped | 1 + tests/corpus/hex-batch | 2 + tests/flutter_compatibility.py | 39 +++ tests/fuzz.c | 11 + tests/oracle.c | 17 + tests/test.c | 38 +++ tests/test_release.py | 110 +++++++ tests/typescript-samples.json | 27 ++ tools/check_conan_center.py | 34 ++ tools/prepare_conan_center.py | 39 +++ tools/release.py | 48 +++ tools/submit_conan_center.py | 103 ++++++ tools/unicode.mjs | 11 + tools/verify_release.py | 40 +++ 46 files changed, 1962 insertions(+), 1 deletion(-) create mode 100644 .editorconfig create mode 100644 .github/ISSUE_TEMPLATE/bug_report.yml create mode 100644 .github/ISSUE_TEMPLATE/config.yml create mode 100644 .github/ISSUE_TEMPLATE/feature_request.yml create mode 100644 .github/pull_request_template.md create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/conan-center.yml create mode 100644 .github/workflows/release.yml create mode 100644 .gitignore create mode 100644 CMakeLists.txt create mode 100644 LICENSE create mode 100644 Makefile create mode 100644 SECURITY.md create mode 100644 cmake/txms.pc.in create mode 100644 cmake/txmsConfig.cmake.in create mode 100644 conanfile.py create mode 100644 include/txms.h create mode 100644 packaging/LICENSE create mode 100644 packaging/conan-center/conanfile.py create mode 100644 src/internal.h create mode 100644 src/txms.c create mode 100644 src/txms_decode.c create mode 100644 src/txms_encode.c create mode 100644 src/txms_hex.c create mode 100644 src/txms_utf16.c create mode 100644 src/unicode_ranges.h create mode 100644 test_package/CMakeLists.txt create mode 100644 test_package/conanfile.py create mode 100644 test_package/test_package.c create mode 100644 tests/VALIDATION.md create mode 100644 tests/compatibility.mjs create mode 100644 tests/corpus/escaped create mode 100644 tests/corpus/hex-batch create mode 100644 tests/flutter_compatibility.py create mode 100644 tests/fuzz.c create mode 100644 tests/oracle.c create mode 100644 tests/test.c create mode 100644 tests/test_release.py create mode 100644 tests/typescript-samples.json create mode 100644 tools/check_conan_center.py create mode 100644 tools/prepare_conan_center.py create mode 100644 tools/release.py create mode 100644 tools/submit_conan_center.py create mode 100644 tools/unicode.mjs create mode 100644 tools/verify_release.py diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..bcc103b --- /dev/null +++ b/.editorconfig @@ -0,0 +1,11 @@ +root = true + +[*] +charset = utf-8 +end_of_line = lf +insert_final_newline = true +indent_style = tab + +[*.{yml,yaml,json}] +indent_style = space +indent_size = 2 diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 0000000..f5eaea9 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,23 @@ +name: Bug report +description: Report a reproducible defect in libtxms +body: + - type: textarea + id: environment + attributes: + label: Environment + description: Commit, OS, compiler, dependency versions, and build options. + validations: + required: true + - type: textarea + id: reproduce + attributes: + label: Reproduction + description: Minimal synthetic input and commands. Remove credentials and private transaction data. + validations: + required: true + - type: textarea + id: result + attributes: + label: Expected and actual results + validations: + required: true diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..3ba13e0 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1 @@ +blank_issues_enabled: false diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml new file mode 100644 index 0000000..dbab269 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -0,0 +1,13 @@ +name: Feature request +description: Propose a specific improvement +body: + - type: textarea + id: usecase + attributes: + label: Use case and expected behavior + validations: + required: true + - type: textarea + id: compatibility + attributes: + label: Protocol evidence and compatibility considerations diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..049ff77 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,7 @@ +## Change + +Describe the concrete problem and resulting behavior. + +## Validation + +Commands run, results, and any remaining limitations. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..86d8e9e --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,78 @@ +name: C library CI +on: + push: + branches: ["**"] + pull_request: + workflow_dispatch: + workflow_call: +permissions: + contents: read +jobs: + build: + strategy: + matrix: + os: [ubuntu-latest, macos-latest] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - run: cmake -S . -B build -DTXMS_SANITIZE=ON -DCMAKE_C_COMPILER=clang + - run: cmake --build build -j2 + - run: ctest --test-dir build --output-on-failure + - uses: actions/setup-node@v4 + with: + node-version: '24.2.0' + - uses: actions/checkout@v4 + with: + repository: bchainhub/txms.js + ref: 53de2be00f10a9439781f90d6a92db418ef2fdef + path: reference-js + - run: npm install && npm test + working-directory: reference-js + - run: node tests/compatibility.mjs build/txms_oracle "$PWD/reference-js/dist/index.js" + flutter: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/checkout@v4 + with: + repository: bchainhub/flutter_txms + ref: 3a593f0c3021007ee21b74932d2d640bc6fff2a3 + path: reference-dart + - uses: subosito/flutter-action@v2 + with: + channel: stable + cache: true + - run: cmake -S . -B build && cmake --build build -j2 + - run: flutter test + working-directory: reference-dart + - run: python3 tests/flutter_compatibility.py build/txms_oracle reference-dart + fuzz: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - run: cmake -S . -B build -DCMAKE_C_COMPILER=clang -DTXMS_SANITIZE=ON -DTXMS_FUZZ=ON + - run: cmake --build build -j2 + - run: build/txms_fuzz tests/corpus -runs=20000 -max_len=2048 + conan: + strategy: + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + shared: ['False', 'True'] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: python -m pip install 'conan>=2.12,<3' + - run: conan profile detect --force + - run: conan create . --build=missing --no-remote -o "libtxms/*:shared=${{ matrix.shared }}" + release-tooling: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: python -m pip install pyyaml + - run: python tests/test_release.py diff --git a/.github/workflows/conan-center.yml b/.github/workflows/conan-center.yml new file mode 100644 index 0000000..aeee49b --- /dev/null +++ b/.github/workflows/conan-center.yml @@ -0,0 +1,59 @@ +name: Submit libtxms to ConanCenter +on: + workflow_call: + inputs: + tag: + required: true + type: string + secrets: + CONAN_CENTER_TOKEN: + required: true + workflow_dispatch: + inputs: + tag: + description: Published stable release tag, for example 0.1.0 + required: true + type: string +permissions: + contents: read +concurrency: + group: conan-center-libtxms + cancel-in-progress: false +jobs: + submit: + if: vars.CONAN_CENTER_FORK != '' + runs-on: ubuntu-latest + env: + TAG: ${{ inputs.tag }} + FORK: ${{ vars.CONAN_CENTER_FORK }} + steps: + - name: Validate release tag + shell: bash + run: '[[ "$TAG" =~ ^(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$ ]]' + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.tag }} + fetch-depth: 0 + persist-credentials: false + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: python -m pip install 'conan>=2.12,<3' + - run: conan profile detect --force + - name: Download and verify published sources + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + run: | + gh release view "$TAG" --json isDraft,isPrerelease --jq 'if .isDraft or .isPrerelease then error("A stable published release is required") else true end' + gh release download "$TAG" --dir dist --pattern "libtxms-${TAG}.tar.gz" --pattern SHA256SUMS --pattern release.json + python tools/verify_release.py --tag "$TAG" --directory dist + - name: Prepare and test recipe + run: | + version="${TAG}" + python tools/prepare_conan_center.py --version "$version" --archive "dist/libtxms-$version.tar.gz" + python tools/check_conan_center.py --version "$version" --archive "dist/libtxms-$version.tar.gz" --recipe release-stage/recipes/libtxms/all + - name: Open or update ConanCenter pull request + env: + GH_TOKEN: ${{ secrets.CONAN_CENTER_TOKEN }} + run: python tools/submit_conan_center.py --fork "$FORK" --version "${TAG}" --stage release-stage diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..18fc0e9 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,70 @@ +name: Release +on: + push: + tags: ['[0-9]*.[0-9]*.[0-9]*'] +permissions: + contents: read +concurrency: + group: release-${{ github.ref }} + cancel-in-progress: false +jobs: + tests: + uses: ./.github/workflows/ci.yml + assets: + needs: tests + runs-on: ubuntu-latest + env: + TAG: ${{ github.ref_name }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + persist-credentials: false + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: python -m pip install 'conan>=2.12,<3' + - run: conan profile detect --force + - name: Verify version and archive the tested commit + run: python tools/release.py --project libtxms --tag "$TAG" + - name: Prepare and test the ConanCenter download recipe + run: | + version="${TAG}" + python tools/prepare_conan_center.py --version "$version" --archive "dist/libtxms-$version.tar.gz" + python tools/check_conan_center.py --version "$version" --archive "dist/libtxms-$version.tar.gz" --recipe release-stage/recipes/libtxms/all + tar -czf "dist/libtxms-$version-conan-center.tar.gz" -C release-stage LICENSE recipes + - name: Checksum all release assets + run: | + cd dist + sha256sum -- *.tar.gz release.json > SHA256SUMS + - uses: actions/upload-artifact@v4 + with: + name: release-assets + path: dist/* + if-no-files-found: error + publish: + needs: [tests, assets] + runs-on: ubuntu-latest + permissions: + contents: write + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + TAG: ${{ github.ref_name }} + steps: + - uses: actions/download-artifact@v4 + with: + name: release-assets + path: dist + - name: Verify downloaded assets + run: cd dist && sha256sum --check SHA256SUMS + - name: Publish the tested release + run: gh release create "$TAG" dist/* --verify-tag --title "$TAG" --generate-notes + conan-center: + needs: publish + if: vars.CONAN_CENTER_FORK != '' + uses: ./.github/workflows/conan-center.yml + with: + tag: ${{ github.ref_name }} + secrets: + CONAN_CENTER_TOKEN: ${{ secrets.CONAN_CENTER_TOKEN }} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..30cee1d --- /dev/null +++ b/.gitignore @@ -0,0 +1,51 @@ +# Build, package and compiler output +/build*/ +/out/ +/dist/ +/release-stage/ +CMakeUserPresets.json +CMakeCache.txt +CMakeFiles/ +compile_commands.json +*.o +*.obj +*.a +*.so +*.so.* +*.dylib +*.dll +*.lib +*.exe +*.pdb +*.gcda +*.gcno +*.profraw +*.profdata + +# Local Conan state; no dependency lock is needed by the current recipes +/conan.lock +/.conan2/ +/test_package/build*/ +/test_package/CMakeUserPresets.json + +# Tool environments and reference checkouts +/.venv/ +/reference-js/ +/reference-dart/ +node_modules/ +__pycache__/ +*.py[cod] +.pytest_cache/ +.DS_Store +*.swp +*.swo + +# Local configuration, storage and fuzz findings +.env +.env.* +!.env.example +*.sqlite3* +*.db* +crash-* +leak-* +timeout-* diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..c12883f --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,50 @@ +cmake_minimum_required(VERSION 3.16) +project(txms VERSION 0.1.0 LANGUAGES C) +include(GNUInstallDirs) +include(CMakePackageConfigHelpers) +include(CTest) +set(CMAKE_POSITION_INDEPENDENT_CODE ON CACHE BOOL "Build position independent code") +option(TXMS_SANITIZE "Address and undefined behavior sanitizers" OFF) +option(TXMS_FUZZ "Build libFuzzer target (Clang)" OFF) +add_library(txms src/txms.c src/txms_hex.c src/txms_utf16.c src/txms_encode.c src/txms_decode.c) +target_include_directories(txms PUBLIC $ $) +target_compile_features(txms PUBLIC c_std_11) +if(MSVC) + target_compile_options(txms PRIVATE /W4 /WX /utf-8) + set_target_properties(txms PROPERTIES WINDOWS_EXPORT_ALL_SYMBOLS ON) +else() + target_compile_options(txms PRIVATE -Wall -Wextra -Wpedantic -Werror) +endif() +set_target_properties(txms PROPERTIES VERSION ${PROJECT_VERSION} SOVERSION 0) +if(TXMS_SANITIZE) + target_compile_options(txms PUBLIC -fsanitize=address,undefined -fno-omit-frame-pointer) + target_link_options(txms PUBLIC -fsanitize=address,undefined) +endif() +if(BUILD_TESTING) + add_executable(txms_test tests/test.c) + target_link_libraries(txms_test PRIVATE txms) + if(MSVC) + target_compile_options(txms_test PRIVATE /W4 /WX /utf-8 /UNDEBUG) + else() + target_compile_options(txms_test PRIVATE -Wall -Wextra -Wpedantic -Werror -UNDEBUG) + endif() + add_test(NAME txms_unit COMMAND txms_test) + add_executable(txms_oracle tests/oracle.c) + target_link_libraries(txms_oracle PRIVATE txms) +endif() +if(TXMS_FUZZ) + target_compile_options(txms PRIVATE -fsanitize=fuzzer-no-link,address,undefined) + add_executable(txms_fuzz tests/fuzz.c) + target_link_libraries(txms_fuzz PRIVATE txms) + target_compile_options(txms_fuzz PRIVATE -fsanitize=fuzzer,address,undefined) + target_link_options(txms_fuzz PRIVATE -fsanitize=fuzzer,address,undefined) +endif() +install(TARGETS txms EXPORT txmsTargets ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) +install(FILES include/txms.h DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) +install(EXPORT txmsTargets NAMESPACE txms:: DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/txms) +file(RELATIVE_PATH TXMS_PC_PREFIX_RELATIVE "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}/pkgconfig" "${CMAKE_INSTALL_PREFIX}") +configure_file(cmake/txms.pc.in txms.pc @ONLY) +configure_package_config_file(cmake/txmsConfig.cmake.in ${CMAKE_CURRENT_BINARY_DIR}/txmsConfig.cmake INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/txms) +write_basic_package_version_file(${CMAKE_CURRENT_BINARY_DIR}/txmsConfigVersion.cmake VERSION ${PROJECT_VERSION} COMPATIBILITY SameMajorVersion) +install(FILES ${CMAKE_CURRENT_BINARY_DIR}/txmsConfig.cmake ${CMAKE_CURRENT_BINARY_DIR}/txmsConfigVersion.cmake DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/txms) +install(FILES ${CMAKE_CURRENT_BINARY_DIR}/txms.pc DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig) diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..d6417b9 --- /dev/null +++ b/LICENSE @@ -0,0 +1,19 @@ +CORE License + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), +to to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +All distribution of the Covered Software in Source Code Form, including any +Modifications and/or Contributions must be disclosed and publicly available. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESSED OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND NON-INFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS BE LIABLE TO ANY CLAIM, DAMAGES OR +OTHER LIABILITIES, WHETHER IN AN ACTION OF A CONTRACT, TORT, OR OTHERWISE, +ARISING FROM, OUT OF, OR IN CONNECTION WITH THE SOFTWARE OR THE USE, OR +OTHER DEALINGS IN THE SOFTWARE. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..6fc9d45 --- /dev/null +++ b/Makefile @@ -0,0 +1,12 @@ +.PHONY: all test sanitize clean +all: + cmake -S . -B build + cmake --build build +test: all + ctest --test-dir build --output-on-failure +sanitize: + cmake -S . -B build-sanitize -DTXMS_SANITIZE=ON + cmake --build build-sanitize + ctest --test-dir build-sanitize --output-on-failure +clean: + rm -rf build build-sanitize diff --git a/README.md b/README.md index ed3802e..93afad3 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,159 @@ # libtxms -TxMS in C + +A standalone C11 implementation of the TxMS codec. No Kamailio, Node, Dart, or runtime Unicode dependency. Licensed under the [CORE License](LICENSE). + +## Build and test + +```sh +cmake -S . -B build +cmake --build build +ctest --test-dir build --output-on-failure +make sanitize +cmake --install build --prefix /usr/local +``` + +Use `-DBUILD_SHARED_LIBS=ON` for a shared library. A header, library, CMake target export and `txms.pc` are installed. Consumers can use `pkg-config --cflags --libs txms`. macOS and Linux are tested. The codec uses standard C11 and has no platform API dependencies. + +```c +#include + +txms_buffer encoded = {0}, decoded = {0}; +if (txms_encode("0x1234abcd", 10, &encoded) == TXMS_OK) { + if (txms_decode(encoded.data, encoded.len, &decoded) == TXMS_OK) { + /* decoded.data contains "0x1234abcd"; decoded.len excludes the NUL. */ + } +} +txms_buffer_free(&decoded); +txms_buffer_free(&encoded); +``` + +All inputs have explicit byte lengths. Outputs are allocated by the library and include an extra NUL sentinel; use the length, especially for UTF-16BE. Initialize each output to `{0}` and free it before reuse. Errors leave outputs empty. Input is capped at 16 MiB; output arithmetic is checked before allocation. APIs are reentrant and own no mutable global state. + +## API and wire behavior + +- `txms_encode`: hexadecimal → UTF-8 TxMS. Optional `0x`/`0X`, either case, nonempty. Odd nibble counts are accepted, matching JavaScript. The first UTF-16 code unit is padded on the left to four hex digits. +- `txms_encode_bytes`: explicit-length binary → TxMS, including embedded zero bytes. It produces the same result as encoding the corresponding hex string. +- `txms_decode`: UTF-8 TxMS → lowercase `0x` hex. Escapes are `~` followed by two U+0100–U+01FF code units. Their low bytes reconstruct the original UTF-16 unit. Surrogates in the transaction are escaped independently. +- `txms_is_hex`: nonempty even hex digits with an optional prefix. No whitespace, embedded NUL or separators. +- `txms_normalize_hex`: strict detection followed by lowercase, prefixed normalization. Preserves leading zeroes. +- `txms_utf16be_to_utf8` / `txms_utf8_to_utf16be`: strict Unicode conversion. Reject unpaired surrogates, overlong UTF-8, truncated input and invalid scalars. +- `txms_transactions`: validates a complete newline-separated batch atomically. Valid hex bypasses TxMS decoding. CRLF delimiters and one terminal LF are accepted; whitespace and BOMs are not stripped. Empty lines, invalid prefixed hex and odd strings consisting entirely of hex digits are rejected. Every final transaction must have a nonempty even number of digits. + +**The reference codec is lossy for leading zeroes.** JavaScript decoding strips every leading zero nibble, so `encode("0001")` decodes to `"0x1"`, and zero decodes to `"0x"`. This implementation preserves that behavior. Binary input is supported, but the original length/leading zeroes cannot be recovered without out-of-band information. The gateway batch API rejects odd or empty decoded results instead of inventing bytes. Use plain normalized hex to preserve leading zeroes for submission. + +Malformed escapes are rejected rather than reproducing JavaScript's `NaN` coercion. Valid encoded data remains byte-for-byte compatible. The special-character table uses Unicode 16.0 categories C and Z, plus U+FFFD and `~`. It is checked in for reproducible behavior; regenerate with `node tools/unicode.mjs` only after reviewing compatibility changes. + +## Reference provenance and cross-language tests + +Sources inspected before implementation: + +- [txms.js](https://github.com/bchainhub/txms.js), commit `53de2be00f10a9439781f90d6a92db418ef2fdef`: primary behavioral reference; CORE License. Its `test/samples.json` is preserved as `tests/typescript-samples.json`. +- [flutter_txms](https://github.com/bchainhub/flutter_txms), commit `3a593f0c3021007ee21b74932d2d640bc6fff2a3`: cross-check. + +The Flutter implementation encodes the selected vectors identically, but its escape decoder retains the `01` prefix of each escape character. For example, `~Āž` decodes to `0x10017e` in Dart and `0x7e` in JavaScript/C. The test records this known upstream divergence; the C library follows TypeScript. Neither reference repository is modified. + +```sh +# Build a separate copy of the pinned TypeScript checkout first (npm install; npm run build). +node tests/compatibility.mjs build/txms_oracle /path/to/txms.js/dist/index.js +# Copies the Flutter checkout into a temporary directory before running Flutter. +python3 tests/flutter_compatibility.py build/txms_oracle /path/to/flutter_txms +``` + +The TypeScript comparison covers all 65,536 BMP code units, 1,000 deterministic randomized byte strings, published samples, odd hex lengths, prefixes and zero padding: 66,543 comparisons. Unit tests also cover invalid encodings, UTF-16 conversion and batch validation. + +```sh +CC=clang cmake -S . -B build-fuzz -DTXMS_FUZZ=ON -DTXMS_SANITIZE=ON +cmake --build build-fuzz +build-fuzz/txms_fuzz -runs=20000 -max_len=2048 +``` + +No transaction structure or signature validation is attempted; those belong to the blockchain node. + +## Conan 2 packages + +A Conan 2 recipe and a separate C consumer test are included. The codec has no +third-party runtime dependencies, so no Conan dependency lockfile is needed or +committed. Release sources are pinned by version and SHA-256 instead. Build, +package-cache, generated preset and local environment files are ignored by Git. + +```sh +python3 -m pip install 'conan>=2.12,<3' +conan profile detect +conan create . --build=missing --no-remote +conan create . --build=missing --no-remote -o 'libtxms/*:shared=True' +# Consume the package from the local Conan cache: +conan install --requires=libtxms/0.1.0 --build=missing -g CMakeDeps -g CMakeToolchain +``` + +Consumers use `find_package(txms CONFIG REQUIRED)` and link `txms::txms`. +Conan packages include the CORE license and attribution. Packaging recipes and +the standalone `test_package` are MIT licensed under [packaging/LICENSE](packaging/LICENSE); +the library remains CORE licensed. `shared` and `fPIC` are supported. CI builds +static/shared packages on Linux, macOS and Windows; Windows packaging checks +run in hosted CI and have not been executed locally. + +## GitHub releases + +[release.yml](.github/workflows/release.yml) runs on stable `MAJOR.MINOR.PATCH` +tags. It calls the complete CI workflow at the same tagged commit, including +sanitizers, compatibility tests, fuzz checks and Conan consumer tests. Publishing +is blocked until every CI job and release-package check succeeds. + +1. Update the version in `CMakeLists.txt`, commit the release changes and push them. +2. Create and push the matching tag, for example: + + ```sh + git tag -a 0.1.0 -m 'libtxms 0.1.0' + git push origin 0.1.0 + ``` + +3. The workflow verifies that the tag matches the CMake version and publishes: + - `libtxms-0.1.0.tar.gz`: source from the exact tested Git commit. + - `libtxms-0.1.0-conan-center.tar.gz`: staged ConanCenter recipe and consumer test. + - `release.json`: version, commit and source checksum. + - `SHA256SUMS`: checksums for all release assets. + +The workflow first builds static/shared ConanCenter packages from the same +archive served locally to exercise download and checksum verification. It then +publishes a GitHub Release with generated notes using the repository's normal +`GITHUB_TOKEN`. Existing releases are never overwritten; rerun a failed job +before publication, or issue a new version after publication. No prebuilt +platform libraries are attached; Conan builds the appropriate package. + +## ConanCenter submission automation + +ConanCenter accepts recipes through reviewed pull requests; its own build +service publishes packages after merge. See the [official contribution guide](https://docs.conan.io/2/tutorial/conan_repositories/conan_center.html). +This repository prepares and submits the recipe; it cannot grant ConanCenter +approval or claim the package is already available there. + +Configure the following **in the libtxms GitHub repository**: + +| Setting | Value | +| --- | --- | +| Repository variable `CONAN_CENTER_FORK` | Your existing fork, e.g. `DataLayerHost/conan-center-index` | +| Actions secret `CONAN_CENTER_TOKEN` | Token able to push to that fork and open PRs in `conan-io/conan-center-index` | + +A classic PAT with `public_repo` is one option for public repositories; use a +service account with the required access and follow your organization's token +policy. The default `GITHUB_TOKEN` cannot write to a separate fork. Never store +the token in source files. A first-time submitter may need to sign ConanCenter's +CLA. ConanCenter maintainers also decide acceptance of the custom CORE license; +the recipe declares `LicenseRef-libtxms-CORE` and includes its complete text. + +When the fork variable is set, successful release publication directly calls +[conan-center.yml](.github/workflows/conan-center.yml). This avoids relying on a +second release event generated by `GITHUB_TOKEN`. The workflow downloads the +published source, checks its checksum and commit metadata, rebuilds static/shared +packages, and opens or updates a recipe PR. It preserves earlier versions and +refuses to replace an existing version's source metadata. Branches are pushed +normally, without force. If the variable is absent, the release still includes +the prepared recipe bundle and automatic PR submission is skipped. + +For a failed submission after publication, run **Actions → Submit libtxms to +ConanCenter → Run workflow**, supplying the existing tag. A missing token fails +submission without changing the published release. This manual path verifies +release assets and reruns package tests too. Until ConanCenter accepts the PR, +use `conan create .` from the matching release checkout. + +Local validation details are in [tests/VALIDATION.md](tests/VALIDATION.md). diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..2f5ea76 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,5 @@ +# Security reporting + +Report vulnerabilities through the repository's **Security → Report a vulnerability** interface when private reporting is enabled. Avoid publishing exploitable input or credentials in public issues. If private reporting is unavailable, ask the repository owner to enable it without disclosing exploit details. + +Only the current main branch is maintained. Sanitize all logs and use synthetic transactions in reports. This first release needs deployment-specific load testing and security review before handling production funds. diff --git a/cmake/txms.pc.in b/cmake/txms.pc.in new file mode 100644 index 0000000..5aadca4 --- /dev/null +++ b/cmake/txms.pc.in @@ -0,0 +1,8 @@ +prefix=${pcfiledir}/@TXMS_PC_PREFIX_RELATIVE@ +libdir=${prefix}/@CMAKE_INSTALL_LIBDIR@ +includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@ +Name: libtxms +Description: TxMS transaction messaging codec +Version: @PROJECT_VERSION@ +Libs: -L${libdir} -ltxms +Cflags: -I${includedir} diff --git a/cmake/txmsConfig.cmake.in b/cmake/txmsConfig.cmake.in new file mode 100644 index 0000000..0463323 --- /dev/null +++ b/cmake/txmsConfig.cmake.in @@ -0,0 +1,3 @@ +@PACKAGE_INIT@ +include("${CMAKE_CURRENT_LIST_DIR}/txmsTargets.cmake") +check_required_components(txms) diff --git a/conanfile.py b/conanfile.py new file mode 100644 index 0000000..971fafb --- /dev/null +++ b/conanfile.py @@ -0,0 +1,66 @@ +# SPDX-License-Identifier: MIT +import os +from conan import ConanFile +from conan.tools.cmake import CMake, CMakeToolchain, cmake_layout +from conan.tools.files import copy, rmdir + +required_conan_version = ">=2.12" + + +class LibtxmsConan(ConanFile): + url = "https://github.com/DataLayerHost/libtxms" + exports_sources = "CMakeLists.txt", "cmake/*", "include/*", "src/*", "LICENSE", "NOTICE" + + name = "libtxms" + package_type = "library" + license = "LicenseRef-libtxms-CORE" + homepage = "https://github.com/DataLayerHost/libtxms" + description = "C11 TxMS transaction messaging codec" + topics = ("sms", "encoding", "blockchain") + settings = "os", "arch", "compiler", "build_type" + options = {"shared": [True, False], "fPIC": [True, False]} + default_options = {"shared": False, "fPIC": True} + + def config_options(self): + if self.settings.os == "Windows": + del self.options.fPIC + + def configure(self): + if self.options.shared: + self.options.rm_safe("fPIC") + self.settings.rm_safe("compiler.cppstd") + self.settings.rm_safe("compiler.libcxx") + + def layout(self): + cmake_layout(self) + + def generate(self): + tc = CMakeToolchain(self) + tc.variables["BUILD_TESTING"] = False + tc.variables["TXMS_SANITIZE"] = False + tc.variables["TXMS_FUZZ"] = False + tc.generate() + + def build(self): + cmake = CMake(self) + cmake.configure() + cmake.build() + + def package(self): + copy(self, "LICENSE", self.source_folder, os.path.join(self.package_folder, "licenses")) + copy(self, "NOTICE", self.source_folder, os.path.join(self.package_folder, "licenses")) + CMake(self).install() + rmdir(self, os.path.join(self.package_folder, "lib", "cmake")) + rmdir(self, os.path.join(self.package_folder, "lib", "pkgconfig")) + + def package_info(self): + self.cpp_info.libs = ["txms"] + self.cpp_info.set_property("cmake_file_name", "txms") + self.cpp_info.set_property("cmake_target_name", "txms::txms") + self.cpp_info.set_property("pkg_config_name", "txms") + + def set_version(self): + if not self.version: + import re + from conan.tools.files import load + self.version = re.search(r"project\(txms VERSION ([0-9.]+)", load(self, os.path.join(self.recipe_folder, "CMakeLists.txt"))).group(1) diff --git a/include/txms.h b/include/txms.h new file mode 100644 index 0000000..a229b7b --- /dev/null +++ b/include/txms.h @@ -0,0 +1,33 @@ +/* SPDX-License-Identifier: LicenseRef-CORE */ +#ifndef TXMS_H +#define TXMS_H +#include +#include +#ifdef __cplusplus +extern "C" { +#endif +#define TXMS_MAX_INPUT (16u * 1024u * 1024u) +typedef struct { uint8_t *data; size_t len; } txms_buffer; +typedef enum { TXMS_OK = 0, TXMS_INVALID = -1, TXMS_LIMIT = -2, TXMS_NOMEM = -3 } txms_status; +/* Initialize outputs to {0}; free successful outputs before reusing them. + * Text inputs/outputs are UTF-8, explicit length, with an extra sentinel NUL. + * encode accepts odd hex lengths like txms.js; is_hex is deliberately stricter. + * decode mirrors JS leading-zero removal and may return odd digits or "0x". + */ +void txms_buffer_free(txms_buffer *buffer); +int txms_is_hex(const void *text, size_t len); +int txms_normalize_hex(const void *text, size_t len, txms_buffer *out); +int txms_encode(const void *hex, size_t len, txms_buffer *out); +int txms_encode_bytes(const void *bytes, size_t len, txms_buffer *out); +int txms_decode(const void *utf8, size_t len, txms_buffer *out); +int txms_utf16be_to_utf8(const void *bytes, size_t len, txms_buffer *out); +int txms_utf8_to_utf16be(const void *text, size_t len, txms_buffer *out); +/* Newline batch normalization. All lines validated before returning. + * CRLF and one terminal newline accepted; no whitespace trimming. + * Output: one nonempty, even-length, lowercase 0x transaction per line. + */ +int txms_transactions(const void *text, size_t len, txms_buffer *out); +#ifdef __cplusplus +} +#endif +#endif diff --git a/packaging/LICENSE b/packaging/LICENSE new file mode 100644 index 0000000..dac29c9 --- /dev/null +++ b/packaging/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 DataLayerHost + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/packaging/conan-center/conanfile.py b/packaging/conan-center/conanfile.py new file mode 100644 index 0000000..0ba13a6 --- /dev/null +++ b/packaging/conan-center/conanfile.py @@ -0,0 +1,62 @@ +# SPDX-License-Identifier: MIT +import os +from conan import ConanFile +from conan.tools.cmake import CMake, CMakeToolchain, cmake_layout +from conan.tools.files import copy, get, rmdir + +required_conan_version = ">=2.12" + + +class LibtxmsConan(ConanFile): + url = "https://github.com/conan-io/conan-center-index" + + name = "libtxms" + package_type = "library" + license = "LicenseRef-libtxms-CORE" + homepage = "https://github.com/DataLayerHost/libtxms" + description = "C11 TxMS transaction messaging codec" + topics = ("sms", "encoding", "blockchain") + settings = "os", "arch", "compiler", "build_type" + options = {"shared": [True, False], "fPIC": [True, False]} + default_options = {"shared": False, "fPIC": True} + + def config_options(self): + if self.settings.os == "Windows": + del self.options.fPIC + + def configure(self): + if self.options.shared: + self.options.rm_safe("fPIC") + self.settings.rm_safe("compiler.cppstd") + self.settings.rm_safe("compiler.libcxx") + + def layout(self): + cmake_layout(self) + + def generate(self): + tc = CMakeToolchain(self) + tc.variables["BUILD_TESTING"] = False + tc.variables["TXMS_SANITIZE"] = False + tc.variables["TXMS_FUZZ"] = False + tc.generate() + + def build(self): + cmake = CMake(self) + cmake.configure() + cmake.build() + + def package(self): + copy(self, "LICENSE", self.source_folder, os.path.join(self.package_folder, "licenses")) + copy(self, "NOTICE", self.source_folder, os.path.join(self.package_folder, "licenses")) + CMake(self).install() + rmdir(self, os.path.join(self.package_folder, "lib", "cmake")) + rmdir(self, os.path.join(self.package_folder, "lib", "pkgconfig")) + + def package_info(self): + self.cpp_info.libs = ["txms"] + self.cpp_info.set_property("cmake_file_name", "txms") + self.cpp_info.set_property("cmake_target_name", "txms::txms") + self.cpp_info.set_property("pkg_config_name", "txms") + + def source(self): + get(self, **self.conan_data["sources"][str(self.version)], strip_root=True) diff --git a/src/internal.h b/src/internal.h new file mode 100644 index 0000000..b94bfcf --- /dev/null +++ b/src/internal.h @@ -0,0 +1,10 @@ +/* SPDX-License-Identifier: LicenseRef-CORE */ +#ifndef TXMS_INTERNAL_H +#define TXMS_INTERNAL_H +#include "txms.h" +int txms_alloc(txms_buffer *b, size_t n); +int txms_nibble(uint8_t c); +int txms_next(const uint8_t *p, size_t n, size_t *i, uint32_t *cp); +size_t txms_put(uint8_t *p, uint32_t cp); +int txms_escape(uint16_t cp); +#endif diff --git a/src/txms.c b/src/txms.c new file mode 100644 index 0000000..c3bad93 --- /dev/null +++ b/src/txms.c @@ -0,0 +1,52 @@ +/* SPDX-License-Identifier: LicenseRef-CORE */ +#include "internal.h" +#include +#include +void txms_buffer_free(txms_buffer *b) +{ + if (b) { free(b->data); b->data = NULL; b->len = 0; } +} +int txms_alloc(txms_buffer *b, size_t n) +{ + if (!b || b->data) return TXMS_INVALID; + if (n > TXMS_MAX_INPUT * 12u) return TXMS_LIMIT; + b->data = malloc(n + 1); + if (!b->data) return TXMS_NOMEM; + b->len = 0; b->data[0] = 0; + return TXMS_OK; +} +int txms_transactions(const void *text, size_t len, txms_buffer *out) +{ + const uint8_t *p = text; + size_t start = 0, used = 0; + int rc; + if (!p || !len || len > TXMS_MAX_INPUT) return TXMS_INVALID; + if ((rc = txms_alloc(out, len * 4 + 3)) != 0) return rc; + while (start < len) { + size_t end = start; + txms_buffer line = {0}, normalized = {0}; + while (end < len && p[end] != '\n') end++; + size_t n = end - start; + if (end < len && n && p[start + n - 1] == '\r') n--; + if (!n) { rc = TXMS_INVALID; goto fail; } + if (txms_is_hex(p + start, n)) rc = txms_normalize_hex(p + start, n, &normalized); + else { + int all_hex = 1; + for (size_t k = 0; k < n; k++) if (txms_nibble(p[start + k]) < 0) all_hex = 0; + if (all_hex || (n >= 2 && p[start] == '0' && (p[start+1] == 'x' || p[start+1] == 'X'))) { rc = TXMS_INVALID; goto fail; } + rc = txms_decode(p + start, n, &line); + if (!rc) rc = txms_normalize_hex(line.data, line.len, &normalized); + } + txms_buffer_free(&line); + if (rc) { txms_buffer_free(&normalized); goto fail; } + if (used) out->data[used++] = '\n'; + memcpy(out->data + used, normalized.data, normalized.len); + used += normalized.len; + txms_buffer_free(&normalized); + start = end + 1; + } + out->len = used; out->data[used] = 0; + return TXMS_OK; +fail: + txms_buffer_free(out); return rc; +} diff --git a/src/txms_decode.c b/src/txms_decode.c new file mode 100644 index 0000000..c9d50cb --- /dev/null +++ b/src/txms_decode.c @@ -0,0 +1,33 @@ +/* SPDX-License-Identifier: LicenseRef-CORE */ +#include "internal.h" +#include +static void unit(txms_buffer *b, uint32_t c) +{ + for (int shift = 12; shift >= 0; shift -= 4) b->data[b->len++] = (uint8_t)"0123456789abcdef"[(c >> shift) & 15]; +} +int txms_decode(const void *text, size_t n, txms_buffer *out) +{ + const uint8_t *p = text; + int rc; + if (!p || n > TXMS_MAX_INPUT) return TXMS_INVALID; + if ((rc = txms_alloc(out, n * 4 + 2))) return rc; + out->data[0] = '0'; out->data[1] = 'x'; out->len = 2; + for (size_t i = 0; i < n;) { + uint32_t c; + if (txms_next(p, n, &i, &c)) goto invalid; + if (c == '~') { + uint32_t a, b; + /* Reject malformed escapes instead of reproducing JS NaN coercion. */ + if (txms_next(p, n, &i, &a) || txms_next(p, n, &i, &b) || a < 0x100 || a > 0x1ff || b < 0x100 || b > 0x1ff) goto invalid; + unit(out, ((a & 255) << 8) | (b & 255)); + } else if (c > 0xffff) { + unit(out, 0xd800 + ((c - 0x10000) >> 10)); unit(out, 0xdc00 + ((c - 0x10000) & 1023)); + } else unit(out, c); + } + size_t first = 2; + while (first < out->len && out->data[first] == '0') first++; + memmove(out->data + 2, out->data + first, out->len - first); + out->len -= first - 2; out->data[out->len] = 0; return 0; +invalid: + txms_buffer_free(out); return TXMS_INVALID; +} diff --git a/src/txms_encode.c b/src/txms_encode.c new file mode 100644 index 0000000..e7bcbf5 --- /dev/null +++ b/src/txms_encode.c @@ -0,0 +1,50 @@ +/* SPDX-License-Identifier: LicenseRef-CORE */ +#include "internal.h" +#include "unicode_ranges.h" +int txms_escape(uint16_t c) +{ + if (c == 0x7e || c == 0xfffd) return 1; + size_t lo = 0, hi = sizeof(txms_ranges) / sizeof(txms_ranges[0]); + while (lo < hi) { + size_t mid = lo + (hi - lo) / 2; + if (c < txms_ranges[mid][0]) hi = mid; + else if (c > txms_ranges[mid][1]) lo = mid + 1; + else return 1; + } + return 0; +} +int txms_encode(const void *hex, size_t n, txms_buffer *out) +{ + const uint8_t *p = hex; + int rc; + if (!p || n > TXMS_MAX_INPUT) return TXMS_INVALID; + if (n >= 2 && p[0] == '0' && (p[1] == 'x' || p[1] == 'X')) { p += 2; n -= 2; } + if (!n) return TXMS_INVALID; + for (size_t i = 0; i < n; i++) if (txms_nibble(p[i]) < 0) return TXMS_INVALID; + if ((rc = txms_alloc(out, ((n + 3) / 4) * 5))) return rc; + /* JS pads the first UTF-16 code unit on the left, including odd nibbles. */ + size_t width = n % 4 ? n % 4 : 4; + for (size_t i = 0; i < n; width = 4) { + uint16_t c = 0; + for (size_t j = 0; j < width; j++) c = (uint16_t)((c << 4) | txms_nibble(p[i++])); + if (txms_escape(c)) { + out->data[out->len++] = '~'; + out->len += txms_put(out->data + out->len, 0x100 + (c >> 8)); + out->len += txms_put(out->data + out->len, 0x100 + (c & 255)); + } else out->len += txms_put(out->data + out->len, c); + } + out->data[out->len] = 0; return 0; +} +int txms_encode_bytes(const void *bytes, size_t n, txms_buffer *out) +{ + const uint8_t *p = bytes; + txms_buffer hex = {0}; + int rc; + if (!p || !n || n > TXMS_MAX_INPUT / 2) return TXMS_INVALID; + if ((rc = txms_alloc(&hex, n * 2))) return rc; + for (size_t i = 0; i < n; i++) { + hex.data[i * 2] = (uint8_t)"0123456789abcdef"[p[i] >> 4]; + hex.data[i * 2 + 1] = (uint8_t)"0123456789abcdef"[p[i] & 15]; + } + rc = txms_encode(hex.data, n * 2, out); txms_buffer_free(&hex); return rc; +} diff --git a/src/txms_hex.c b/src/txms_hex.c new file mode 100644 index 0000000..2c32657 --- /dev/null +++ b/src/txms_hex.c @@ -0,0 +1,30 @@ +/* SPDX-License-Identifier: LicenseRef-CORE */ +#include "internal.h" +int txms_nibble(uint8_t c) +{ + if (c >= '0' && c <= '9') return c - '0'; + if (c >= 'a' && c <= 'f') return c - 'a' + 10; + if (c >= 'A' && c <= 'F') return c - 'A' + 10; + return -1; +} +int txms_is_hex(const void *text, size_t n) +{ + const uint8_t *p = text; + if (!p || n > TXMS_MAX_INPUT) return 0; + if (n >= 2 && p[0] == '0' && (p[1] == 'x' || p[1] == 'X')) { p += 2; n -= 2; } + if (!n || n % 2) return 0; + for (size_t i = 0; i < n; i++) if (txms_nibble(p[i]) < 0) return 0; + return 1; +} +int txms_normalize_hex(const void *text, size_t n, txms_buffer *out) +{ + const uint8_t *p = text; + int rc; + if (!txms_is_hex(p, n)) return TXMS_INVALID; + if (n >= 2 && p[0] == '0' && (p[1] == 'x' || p[1] == 'X')) { p += 2; n -= 2; } + if ((rc = txms_alloc(out, n + 2))) return rc; + out->data[0] = '0'; out->data[1] = 'x'; + for (size_t i = 0; i < n; i++) out->data[i + 2] = (uint8_t)"0123456789abcdef"[txms_nibble(p[i])]; + out->len = n + 2; out->data[out->len] = 0; + return TXMS_OK; +} diff --git a/src/txms_utf16.c b/src/txms_utf16.c new file mode 100644 index 0000000..f91b197 --- /dev/null +++ b/src/txms_utf16.c @@ -0,0 +1,67 @@ +/* SPDX-License-Identifier: LicenseRef-CORE */ +#include "internal.h" +int txms_next(const uint8_t *p, size_t n, size_t *i, uint32_t *cp) +{ + uint32_t c, min; + unsigned extra; + if (*i >= n) return TXMS_INVALID; + c = p[(*i)++]; + if (c < 128) { *cp = c; return 0; } + if (c >= 0xc2 && c <= 0xdf) { extra = 1; min = 0x80; c &= 31; } + else if (c >= 0xe0 && c <= 0xef) { extra = 2; min = 0x800; c &= 15; } + else if (c >= 0xf0 && c <= 0xf4) { extra = 3; min = 0x10000; c &= 7; } + else return TXMS_INVALID; + if (n - *i < extra) return TXMS_INVALID; + while (extra--) { + uint8_t b = p[(*i)++]; + if ((b & 0xc0) != 0x80) return TXMS_INVALID; + c = (c << 6) | (b & 63); + } + if (c < min || c > 0x10ffff || (c >= 0xd800 && c <= 0xdfff)) return TXMS_INVALID; + *cp = c; return 0; +} +size_t txms_put(uint8_t *p, uint32_t c) +{ + if (c < 128) { p[0] = (uint8_t)c; return 1; } + if (c < 0x800) { p[0] = (uint8_t)(0xc0 | (c >> 6)); p[1] = (uint8_t)(0x80 | (c & 63)); return 2; } + if (c < 0x10000) { p[0] = (uint8_t)(0xe0 | (c >> 12)); p[1] = (uint8_t)(0x80 | ((c >> 6) & 63)); p[2] = (uint8_t)(0x80 | (c & 63)); return 3; } + p[0] = (uint8_t)(0xf0 | (c >> 18)); p[1] = (uint8_t)(0x80 | ((c >> 12) & 63)); p[2] = (uint8_t)(0x80 | ((c >> 6) & 63)); p[3] = (uint8_t)(0x80 | (c & 63)); return 4; +} +int txms_utf16be_to_utf8(const void *bytes, size_t n, txms_buffer *out) +{ + const uint8_t *p = bytes; + int rc; + if (!p || n % 2 || n > TXMS_MAX_INPUT) return TXMS_INVALID; + if ((rc = txms_alloc(out, n * 2))) return rc; + for (size_t i = 0; i < n; i += 2) { + uint32_t c = ((uint32_t)p[i] << 8) | p[i + 1]; + if (c >= 0xd800 && c <= 0xdbff) { + if (n - i < 4) goto invalid; + uint32_t lo = ((uint32_t)p[i + 2] << 8) | p[i + 3]; + if (lo < 0xdc00 || lo > 0xdfff) goto invalid; + c = 0x10000 + ((c - 0xd800) << 10) + lo - 0xdc00; i += 2; + } else if (c >= 0xdc00 && c <= 0xdfff) goto invalid; + out->len += txms_put(out->data + out->len, c); + } + out->data[out->len] = 0; return 0; +invalid: + txms_buffer_free(out); return TXMS_INVALID; +} +int txms_utf8_to_utf16be(const void *text, size_t n, txms_buffer *out) +{ + const uint8_t *p = text; + int rc; + if (!p || n > TXMS_MAX_INPUT) return TXMS_INVALID; + if ((rc = txms_alloc(out, n * 2))) return rc; + for (size_t i = 0; i < n;) { + uint32_t c; + if (txms_next(p, n, &i, &c)) { txms_buffer_free(out); return TXMS_INVALID; } + if (c > 0xffff) { + uint32_t hi = 0xd800 + ((c - 0x10000) >> 10); + out->data[out->len++] = (uint8_t)(hi >> 8); out->data[out->len++] = (uint8_t)hi; + c = 0xdc00 + ((c - 0x10000) & 1023); + } + out->data[out->len++] = (uint8_t)(c >> 8); out->data[out->len++] = (uint8_t)c; + } + out->data[out->len] = 0; return 0; +} diff --git a/src/unicode_ranges.h b/src/unicode_ranges.h new file mode 100644 index 0000000..ccfc90c --- /dev/null +++ b/src/unicode_ranges.h @@ -0,0 +1,348 @@ +/* Generated from Node v24.2.0, Unicode 16.0. */ +static const uint16_t txms_ranges[][2] = { + {0x0, 0x20}, + {0x7f, 0xa0}, + {0xad, 0xad}, + {0x378, 0x379}, + {0x380, 0x383}, + {0x38b, 0x38b}, + {0x38d, 0x38d}, + {0x3a2, 0x3a2}, + {0x530, 0x530}, + {0x557, 0x558}, + {0x58b, 0x58c}, + {0x590, 0x590}, + {0x5c8, 0x5cf}, + {0x5eb, 0x5ee}, + {0x5f5, 0x605}, + {0x61c, 0x61c}, + {0x6dd, 0x6dd}, + {0x70e, 0x70f}, + {0x74b, 0x74c}, + {0x7b2, 0x7bf}, + {0x7fb, 0x7fc}, + {0x82e, 0x82f}, + {0x83f, 0x83f}, + {0x85c, 0x85d}, + {0x85f, 0x85f}, + {0x86b, 0x86f}, + {0x88f, 0x896}, + {0x8e2, 0x8e2}, + {0x984, 0x984}, + {0x98d, 0x98e}, + {0x991, 0x992}, + {0x9a9, 0x9a9}, + {0x9b1, 0x9b1}, + {0x9b3, 0x9b5}, + {0x9ba, 0x9bb}, + {0x9c5, 0x9c6}, + {0x9c9, 0x9ca}, + {0x9cf, 0x9d6}, + {0x9d8, 0x9db}, + {0x9de, 0x9de}, + {0x9e4, 0x9e5}, + {0x9ff, 0xa00}, + {0xa04, 0xa04}, + {0xa0b, 0xa0e}, + {0xa11, 0xa12}, + {0xa29, 0xa29}, + {0xa31, 0xa31}, + {0xa34, 0xa34}, + {0xa37, 0xa37}, + {0xa3a, 0xa3b}, + {0xa3d, 0xa3d}, + {0xa43, 0xa46}, + {0xa49, 0xa4a}, + {0xa4e, 0xa50}, + {0xa52, 0xa58}, + {0xa5d, 0xa5d}, + {0xa5f, 0xa65}, + {0xa77, 0xa80}, + {0xa84, 0xa84}, + {0xa8e, 0xa8e}, + {0xa92, 0xa92}, + {0xaa9, 0xaa9}, + {0xab1, 0xab1}, + {0xab4, 0xab4}, + {0xaba, 0xabb}, + {0xac6, 0xac6}, + {0xaca, 0xaca}, + {0xace, 0xacf}, + {0xad1, 0xadf}, + {0xae4, 0xae5}, + {0xaf2, 0xaf8}, + {0xb00, 0xb00}, + {0xb04, 0xb04}, + {0xb0d, 0xb0e}, + {0xb11, 0xb12}, + {0xb29, 0xb29}, + {0xb31, 0xb31}, + {0xb34, 0xb34}, + {0xb3a, 0xb3b}, + {0xb45, 0xb46}, + {0xb49, 0xb4a}, + {0xb4e, 0xb54}, + {0xb58, 0xb5b}, + {0xb5e, 0xb5e}, + {0xb64, 0xb65}, + {0xb78, 0xb81}, + {0xb84, 0xb84}, + {0xb8b, 0xb8d}, + {0xb91, 0xb91}, + {0xb96, 0xb98}, + {0xb9b, 0xb9b}, + {0xb9d, 0xb9d}, + {0xba0, 0xba2}, + {0xba5, 0xba7}, + {0xbab, 0xbad}, + {0xbba, 0xbbd}, + {0xbc3, 0xbc5}, + {0xbc9, 0xbc9}, + {0xbce, 0xbcf}, + {0xbd1, 0xbd6}, + {0xbd8, 0xbe5}, + {0xbfb, 0xbff}, + {0xc0d, 0xc0d}, + {0xc11, 0xc11}, + {0xc29, 0xc29}, + {0xc3a, 0xc3b}, + {0xc45, 0xc45}, + {0xc49, 0xc49}, + {0xc4e, 0xc54}, + {0xc57, 0xc57}, + {0xc5b, 0xc5c}, + {0xc5e, 0xc5f}, + {0xc64, 0xc65}, + {0xc70, 0xc76}, + {0xc8d, 0xc8d}, + {0xc91, 0xc91}, + {0xca9, 0xca9}, + {0xcb4, 0xcb4}, + {0xcba, 0xcbb}, + {0xcc5, 0xcc5}, + {0xcc9, 0xcc9}, + {0xcce, 0xcd4}, + {0xcd7, 0xcdc}, + {0xcdf, 0xcdf}, + {0xce4, 0xce5}, + {0xcf0, 0xcf0}, + {0xcf4, 0xcff}, + {0xd0d, 0xd0d}, + {0xd11, 0xd11}, + {0xd45, 0xd45}, + {0xd49, 0xd49}, + {0xd50, 0xd53}, + {0xd64, 0xd65}, + {0xd80, 0xd80}, + {0xd84, 0xd84}, + {0xd97, 0xd99}, + {0xdb2, 0xdb2}, + {0xdbc, 0xdbc}, + {0xdbe, 0xdbf}, + {0xdc7, 0xdc9}, + {0xdcb, 0xdce}, + {0xdd5, 0xdd5}, + {0xdd7, 0xdd7}, + {0xde0, 0xde5}, + {0xdf0, 0xdf1}, + {0xdf5, 0xe00}, + {0xe3b, 0xe3e}, + {0xe5c, 0xe80}, + {0xe83, 0xe83}, + {0xe85, 0xe85}, + {0xe8b, 0xe8b}, + {0xea4, 0xea4}, + {0xea6, 0xea6}, + {0xebe, 0xebf}, + {0xec5, 0xec5}, + {0xec7, 0xec7}, + {0xecf, 0xecf}, + {0xeda, 0xedb}, + {0xee0, 0xeff}, + {0xf48, 0xf48}, + {0xf6d, 0xf70}, + {0xf98, 0xf98}, + {0xfbd, 0xfbd}, + {0xfcd, 0xfcd}, + {0xfdb, 0xfff}, + {0x10c6, 0x10c6}, + {0x10c8, 0x10cc}, + {0x10ce, 0x10cf}, + {0x1249, 0x1249}, + {0x124e, 0x124f}, + {0x1257, 0x1257}, + {0x1259, 0x1259}, + {0x125e, 0x125f}, + {0x1289, 0x1289}, + {0x128e, 0x128f}, + {0x12b1, 0x12b1}, + {0x12b6, 0x12b7}, + {0x12bf, 0x12bf}, + {0x12c1, 0x12c1}, + {0x12c6, 0x12c7}, + {0x12d7, 0x12d7}, + {0x1311, 0x1311}, + {0x1316, 0x1317}, + {0x135b, 0x135c}, + {0x137d, 0x137f}, + {0x139a, 0x139f}, + {0x13f6, 0x13f7}, + {0x13fe, 0x13ff}, + {0x1680, 0x1680}, + {0x169d, 0x169f}, + {0x16f9, 0x16ff}, + {0x1716, 0x171e}, + {0x1737, 0x173f}, + {0x1754, 0x175f}, + {0x176d, 0x176d}, + {0x1771, 0x1771}, + {0x1774, 0x177f}, + {0x17de, 0x17df}, + {0x17ea, 0x17ef}, + {0x17fa, 0x17ff}, + {0x180e, 0x180e}, + {0x181a, 0x181f}, + {0x1879, 0x187f}, + {0x18ab, 0x18af}, + {0x18f6, 0x18ff}, + {0x191f, 0x191f}, + {0x192c, 0x192f}, + {0x193c, 0x193f}, + {0x1941, 0x1943}, + {0x196e, 0x196f}, + {0x1975, 0x197f}, + {0x19ac, 0x19af}, + {0x19ca, 0x19cf}, + {0x19db, 0x19dd}, + {0x1a1c, 0x1a1d}, + {0x1a5f, 0x1a5f}, + {0x1a7d, 0x1a7e}, + {0x1a8a, 0x1a8f}, + {0x1a9a, 0x1a9f}, + {0x1aae, 0x1aaf}, + {0x1acf, 0x1aff}, + {0x1b4d, 0x1b4d}, + {0x1bf4, 0x1bfb}, + {0x1c38, 0x1c3a}, + {0x1c4a, 0x1c4c}, + {0x1c8b, 0x1c8f}, + {0x1cbb, 0x1cbc}, + {0x1cc8, 0x1ccf}, + {0x1cfb, 0x1cff}, + {0x1f16, 0x1f17}, + {0x1f1e, 0x1f1f}, + {0x1f46, 0x1f47}, + {0x1f4e, 0x1f4f}, + {0x1f58, 0x1f58}, + {0x1f5a, 0x1f5a}, + {0x1f5c, 0x1f5c}, + {0x1f5e, 0x1f5e}, + {0x1f7e, 0x1f7f}, + {0x1fb5, 0x1fb5}, + {0x1fc5, 0x1fc5}, + {0x1fd4, 0x1fd5}, + {0x1fdc, 0x1fdc}, + {0x1ff0, 0x1ff1}, + {0x1ff5, 0x1ff5}, + {0x1fff, 0x200f}, + {0x2028, 0x202f}, + {0x205f, 0x206f}, + {0x2072, 0x2073}, + {0x208f, 0x208f}, + {0x209d, 0x209f}, + {0x20c1, 0x20cf}, + {0x20f1, 0x20ff}, + {0x218c, 0x218f}, + {0x242a, 0x243f}, + {0x244b, 0x245f}, + {0x2b74, 0x2b75}, + {0x2b96, 0x2b96}, + {0x2cf4, 0x2cf8}, + {0x2d26, 0x2d26}, + {0x2d28, 0x2d2c}, + {0x2d2e, 0x2d2f}, + {0x2d68, 0x2d6e}, + {0x2d71, 0x2d7e}, + {0x2d97, 0x2d9f}, + {0x2da7, 0x2da7}, + {0x2daf, 0x2daf}, + {0x2db7, 0x2db7}, + {0x2dbf, 0x2dbf}, + {0x2dc7, 0x2dc7}, + {0x2dcf, 0x2dcf}, + {0x2dd7, 0x2dd7}, + {0x2ddf, 0x2ddf}, + {0x2e5e, 0x2e7f}, + {0x2e9a, 0x2e9a}, + {0x2ef4, 0x2eff}, + {0x2fd6, 0x2fef}, + {0x3000, 0x3000}, + {0x3040, 0x3040}, + {0x3097, 0x3098}, + {0x3100, 0x3104}, + {0x3130, 0x3130}, + {0x318f, 0x318f}, + {0x31e6, 0x31ee}, + {0x321f, 0x321f}, + {0xa48d, 0xa48f}, + {0xa4c7, 0xa4cf}, + {0xa62c, 0xa63f}, + {0xa6f8, 0xa6ff}, + {0xa7ce, 0xa7cf}, + {0xa7d2, 0xa7d2}, + {0xa7d4, 0xa7d4}, + {0xa7dd, 0xa7f1}, + {0xa82d, 0xa82f}, + {0xa83a, 0xa83f}, + {0xa878, 0xa87f}, + {0xa8c6, 0xa8cd}, + {0xa8da, 0xa8df}, + {0xa954, 0xa95e}, + {0xa97d, 0xa97f}, + {0xa9ce, 0xa9ce}, + {0xa9da, 0xa9dd}, + {0xa9ff, 0xa9ff}, + {0xaa37, 0xaa3f}, + {0xaa4e, 0xaa4f}, + {0xaa5a, 0xaa5b}, + {0xaac3, 0xaada}, + {0xaaf7, 0xab00}, + {0xab07, 0xab08}, + {0xab0f, 0xab10}, + {0xab17, 0xab1f}, + {0xab27, 0xab27}, + {0xab2f, 0xab2f}, + {0xab6c, 0xab6f}, + {0xabee, 0xabef}, + {0xabfa, 0xabff}, + {0xd7a4, 0xd7af}, + {0xd7c7, 0xd7ca}, + {0xd7fc, 0xf8ff}, + {0xfa6e, 0xfa6f}, + {0xfada, 0xfaff}, + {0xfb07, 0xfb12}, + {0xfb18, 0xfb1c}, + {0xfb37, 0xfb37}, + {0xfb3d, 0xfb3d}, + {0xfb3f, 0xfb3f}, + {0xfb42, 0xfb42}, + {0xfb45, 0xfb45}, + {0xfbc3, 0xfbd2}, + {0xfd90, 0xfd91}, + {0xfdc8, 0xfdce}, + {0xfdd0, 0xfdef}, + {0xfe1a, 0xfe1f}, + {0xfe53, 0xfe53}, + {0xfe67, 0xfe67}, + {0xfe6c, 0xfe6f}, + {0xfe75, 0xfe75}, + {0xfefd, 0xff00}, + {0xffbf, 0xffc1}, + {0xffc8, 0xffc9}, + {0xffd0, 0xffd1}, + {0xffd8, 0xffd9}, + {0xffdd, 0xffdf}, + {0xffe7, 0xffe7}, + {0xffef, 0xfffb}, + {0xfffe, 0xffff}, +}; diff --git a/test_package/CMakeLists.txt b/test_package/CMakeLists.txt new file mode 100644 index 0000000..d97b8d2 --- /dev/null +++ b/test_package/CMakeLists.txt @@ -0,0 +1,5 @@ +cmake_minimum_required(VERSION 3.16) +project(txms_consumer LANGUAGES C) +find_package(txms CONFIG REQUIRED) +add_executable(test_package test_package.c) +target_link_libraries(test_package PRIVATE txms::txms) diff --git a/test_package/conanfile.py b/test_package/conanfile.py new file mode 100644 index 0000000..cae92aa --- /dev/null +++ b/test_package/conanfile.py @@ -0,0 +1,26 @@ +# SPDX-License-Identifier: MIT +import os +from conan import ConanFile +from conan.tools.build import can_run +from conan.tools.cmake import CMake, cmake_layout + + +class LibtxmsTest(ConanFile): + settings = "os", "arch", "compiler", "build_type" + generators = "CMakeDeps", "CMakeToolchain", "VirtualRunEnv" + test_type = "explicit" + + def requirements(self): + self.requires(self.tested_reference_str) + + def layout(self): + cmake_layout(self) + + def build(self): + cmake = CMake(self) + cmake.configure() + cmake.build() + + def test(self): + if can_run(self): + self.run(os.path.join(self.cpp.build.bindir, "test_package"), env="conanrun") diff --git a/test_package/test_package.c b/test_package/test_package.c new file mode 100644 index 0000000..2fcf5ba --- /dev/null +++ b/test_package/test_package.c @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: MIT */ +#include +#include +int main(void) +{ + txms_buffer encoded = {0}, decoded = {0}; + int rc = 1; + if (txms_encode("0x1234abcd", 10, &encoded) != TXMS_OK) goto done; + if (txms_decode(encoded.data, encoded.len, &decoded) != TXMS_OK) goto done; + if (decoded.len == 10 && !memcmp(decoded.data, "0x1234abcd", 10)) rc = 0; +done: + txms_buffer_free(&encoded); + txms_buffer_free(&decoded); + return rc; +} diff --git a/tests/VALIDATION.md b/tests/VALIDATION.md new file mode 100644 index 0000000..f5ca2b4 --- /dev/null +++ b/tests/VALIDATION.md @@ -0,0 +1,30 @@ +# Validation report — 2026-09-28 + +- macOS arm64 / Apple Clang 21: unit tests, ASan and UBSan passed. +- Debian 12 arm64 / Clang 14: unit tests, ASan and UBSan passed through the gateway build. +- 66,543 byte-for-byte TypeScript encode/decode comparisons passed (Node 24.2.0, Unicode 16). +- All 65,536 BMP values also exercised by the C unit tests. +- Flutter cross-check: six matching encodings, two matching decodes and four known escape-decoding divergences verified. +- Reference suites in temporary copies: 68 JavaScript tests and 35 Flutter tests passed. +- Seeded decoder/encoder/UTF-16/batch libFuzzer target: 20,000 iterations, no sanitizer findings. +- Shared library installation, relocatable pkg-config metadata and an independent CMake find_package consumer verified on macOS. +- GitHub workflow and issue-template YAML parsed successfully; hosted CI has not yet run. + +The original TypeScript and Flutter repositories were not modified. The reference +codec loses leading zero nibbles; libtxms preserves that behavior and documents +its effect on binary round trips. Malformed escape inputs are deliberately rejected. + +## Release and Conan automation follow-up + +- GitHub Actions workflows passed `actionlint` 1.7.12 and YAML parsing. +- Release tooling tests passed: tag/version matching, exact-commit archives, + dirty-tree rejection and dependency metadata. +- libtxms Conan 2.32.0 static/shared packages and their separate C consumer passed + on macOS arm64 and Debian 12 arm64. The staged ConanCenter download recipe also + passed both configurations on both operating systems using the same checksummed + archive served by a local HTTP server. +- ConanCenter metadata tests verify preserved older versions, idempotent updates, + refusal to replace a version checksum, and tampered-archive rejection. +- macOS codec and gateway sanitizer suites passed after the CMake packaging changes. +- No release or upstream ConanCenter PR was published. Hosted Windows packaging + and the actual GitHub publication steps remain to be exercised by CI after push. diff --git a/tests/compatibility.mjs b/tests/compatibility.mjs new file mode 100644 index 0000000..6506f2d --- /dev/null +++ b/tests/compatibility.mjs @@ -0,0 +1,26 @@ +import assert from 'node:assert/strict'; +import {readFileSync} from 'node:fs'; +import {pathToFileURL} from 'node:url'; +import {spawnSync} from 'node:child_process'; +const [binary, reference] = process.argv.slice(2); +if (!binary || !reference) throw new Error('Usage: node tests/compatibility.mjs build/txms_oracle /path/to/txms.js/dist/index.js'); +const {default: txms} = await import(pathToFileURL(reference)); +const samples = JSON.parse(readFileSync(new URL('./typescript-samples.json', import.meta.url))); +const vectors = samples.valid.map(s => s.hex); +for (let i=0; i<65536; i++) vectors.push(i.toString(16).padStart(4,'0')); +let seed=123456789; +for (let j=0; j<1000; j++) { + let hex=''; + for (let k=0; k>>0; hex+=(seed&255).toString(16).padStart(2,'0'); } + vectors.push(hex); +} +vectors.push('f','00001','0XABC','00000000','48656c6c6f20576f726c64'); +const child=spawnSync(binary,[],{input:vectors.join('\n')+'\n',encoding:'utf8',maxBuffer:32*1024*1024}); +assert.equal(child.status,0,child.stderr); +const lines=child.stdout.trimEnd().split('\n'); assert.equal(lines.length,vectors.length); +for (let i=0; i +#include +int main(void) +{ + char line[65536]; + while (fgets(line, sizeof(line), stdin)) { + txms_buffer b = {0}, d = {0}; + size_t n = strcspn(line, "\r\n"); + if (txms_encode(line, n, &b) || txms_decode(b.data, b.len, &d)) return 1; + for (size_t i = 0; i < b.len; i++) printf("%02x", b.data[i]); + printf("\t%.*s\n", (int)d.len, d.data); + txms_buffer_free(&b); txms_buffer_free(&d); + } + return ferror(stdin) ? 1 : 0; +} diff --git a/tests/test.c b/tests/test.c new file mode 100644 index 0000000..556835e --- /dev/null +++ b/tests/test.c @@ -0,0 +1,38 @@ +/* SPDX-License-Identifier: LicenseRef-CORE */ +#include "txms.h" +#include +#include +int main(void) +{ + txms_buffer b = {0}, c = {0}; + assert(txms_is_hex("0XAb00", 6)); + assert(!txms_is_hex("0x", 2)); assert(!txms_is_hex("abc", 3)); + assert(!txms_is_hex("ab\0c", 4)); assert(!txms_is_hex(" ab ", 4)); + assert(txms_normalize_hex("0XAb00", 6, &b) == 0); + assert(b.len == 6 && !memcmp(b.data, "0xab00", 6)); txms_buffer_free(&b); + assert(txms_encode("0000007ed800ffff", 16, &b) == 0); + assert(txms_decode(b.data, b.len, &c) == 0); + assert(!strcmp((char *)c.data, "0x7ed800ffff")); + txms_buffer_free(&b); txms_buffer_free(&c); + assert(txms_decode("~", 1, &b) == TXMS_INVALID); + assert(txms_decode("~aa", 3, &b) == TXMS_INVALID); + assert(txms_decode("\xc0\xaf", 2, &b) == TXMS_INVALID); + assert(txms_utf16be_to_utf8("\xd8\x00", 2, &b) == TXMS_INVALID); + assert(txms_utf16be_to_utf8("\xd8\x3d\xde\x00", 4, &b) == 0); + assert(txms_utf8_to_utf16be(b.data, b.len, &c) == 0); + assert(c.len == 4 && !memcmp(c.data, "\xd8\x3d\xde\x00", 4)); + txms_buffer_free(&b); txms_buffer_free(&c); + assert(txms_transactions("0XAB\r\n1234\n", 11, &b) == 0); + assert(!strcmp((char *)b.data, "0xab\n0x1234")); txms_buffer_free(&b); + assert(txms_transactions("ab\n\ncd", 6, &b) == TXMS_INVALID); + assert(txms_transactions("abc", 3, &b) == TXMS_INVALID); + assert(txms_transactions("0xGG", 4, &b) == TXMS_INVALID); + assert(txms_transactions("~Āā", 5, &b) == TXMS_INVALID); /* JS strips to odd 0x1. */ + for (unsigned v = 0; v < 65536; v++) { + uint8_t bytes[2] = {(uint8_t)(v >> 8), (uint8_t)v}; + assert(!txms_encode_bytes(bytes, 2, &b)); + assert(!txms_decode(b.data, b.len, &c)); + txms_buffer_free(&b); txms_buffer_free(&c); + } + return 0; +} diff --git a/tests/test_release.py b/tests/test_release.py new file mode 100644 index 0000000..0b018b6 --- /dev/null +++ b/tests/test_release.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 +import hashlib +from pathlib import Path +import subprocess +import sys +import tarfile +import tempfile +import unittest + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tools")) +from release import build_release + + +class ReleaseTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory(prefix="txms-release-test-") + self.root = Path(self.temp.name) + self.git("init", "-q") + self.git("config", "user.email", "test@example.invalid") + self.git("config", "user.name", "Release test") + (self.root / "CMakeLists.txt").write_text("project(txms VERSION 0.1.0 LANGUAGES C)\n") + (self.root / "LICENSE").write_text("Test fixture license\n") + (self.root / "LIBTXMS_VERSION").write_text("0.1.0\n") + self.git("add", ".") + self.git("commit", "-qm", "fixture") + self.git("tag", "0.1.0") + + def tearDown(self): + self.temp.cleanup() + + def git(self, *args): + return subprocess.check_output(["git", "-C", str(self.root), *args], text=True).strip() + + def test_archive_is_tagged_tree_with_checksums(self): + (self.root / "conan.lock").write_text("must not enter release\n") + out = self.root / "dist" + metadata = build_release(self.root, "libtxms", "0.1.0", out) + archive = out / "libtxms-0.1.0.tar.gz" + with tarfile.open(archive) as source: + self.assertIn("libtxms-0.1.0/LICENSE", source.getnames()) + self.assertNotIn("libtxms-0.1.0/conan.lock", source.getnames()) + self.assertEqual(metadata["commit"], self.git("rev-parse", "HEAD")) + self.assertEqual(metadata["source_sha256"], hashlib.sha256(archive.read_bytes()).hexdigest()) + + def test_gateway_records_tested_dependency(self): + metadata = build_release(self.root, "kamailio-txms", "0.1.0", self.root / "dist") + self.assertEqual(metadata["libtxms_tag"], "0.1.0") + + def test_mismatched_or_unsafe_version_rejected(self): + self.git("tag", "0.2.0") + for tag in ("0.2.0", "v0.1.0", "0.1.0-rc1", "0.1.0;echo bad", "00.1.0"): + with self.subTest(tag=tag), self.assertRaises(ValueError): + build_release(self.root, "libtxms", tag, self.root / "dist") + + def test_dirty_tracked_file_rejected(self): + (self.root / "LICENSE").write_text("changed\n") + with self.assertRaises(ValueError): + build_release(self.root, "libtxms", "0.1.0", self.root / "dist") + + def test_checkout_must_match_tag(self): + (self.root / "LICENSE").write_text("changed\n") + self.git("commit", "-qam", "changed") + with self.assertRaises(ValueError): + build_release(self.root, "libtxms", "0.1.0", self.root / "dist") + + +if (ROOT / "tools/prepare_conan_center.py").exists(): + import yaml + from prepare_conan_center import prepare + from submit_conan_center import merge_recipe + from verify_release import verify + + class ConanCenterTests(unittest.TestCase): + setUp = ReleaseTests.setUp + tearDown = ReleaseTests.tearDown + git = ReleaseTests.git + def test_release_verification_rejects_tampering(self): + out = self.root / "dist" + metadata = build_release(self.root, "libtxms", "0.1.0", out) + with (out / "SHA256SUMS").open("a") as checksums: + checksums.write(hashlib.sha256((out / "release.json").read_bytes()).hexdigest() + " release.json\n") + self.assertEqual(verify(out, "0.1.0", metadata["commit"]), metadata) + (out / "libtxms-0.1.0.tar.gz").write_bytes(b"tampered") + with self.assertRaises(ValueError): + verify(out, "0.1.0", metadata["commit"]) + + def test_conan_metadata_preserves_existing_versions(self): + out = self.root / "dist" + build_release(self.root, "libtxms", "0.1.0", out) + stage = self.root / "stage" + prepare(ROOT, "0.1.0", out / "libtxms-0.1.0.tar.gz", stage) + destination = self.root / "index" + old = destination / "recipes/libtxms/all" + old.mkdir(parents=True) + (old / "conandata.yml").write_text(yaml.safe_dump({"sources": {"0.0.9": {"url": "https://example.invalid/old", "sha256": "a"*64}}})) + (old.parent / "config.yml").write_text(yaml.safe_dump({"versions": {"0.0.9": {"folder": "all"}}})) + merge_recipe(stage, destination, "0.1.0") + before = (old / "conandata.yml").read_bytes() + self.assertEqual(set(yaml.safe_load(before)["sources"]), {"0.0.9", "0.1.0"}) + merge_recipe(stage, destination, "0.1.0") + self.assertEqual(before, (old / "conandata.yml").read_bytes()) + data = yaml.safe_load((stage / "recipes/libtxms/all/conandata.yml").read_text()) + data["sources"]["0.1.0"]["sha256"] = "b"*64 + (stage / "recipes/libtxms/all/conandata.yml").write_text(yaml.safe_dump(data)) + with self.assertRaises(ValueError): + merge_recipe(stage, destination, "0.1.0") + +if __name__ == "__main__": + unittest.main() diff --git a/tests/typescript-samples.json b/tests/typescript-samples.json new file mode 100644 index 0000000..7a0500d --- /dev/null +++ b/tests/typescript-samples.json @@ -0,0 +1,27 @@ +{ + "valid": [ + { + "hex": "0xf8d880843b9aca008252080396ab2896c9af969b11394e3f0be750da75d0076e1d1afd880de0b6b3a764000080b8ab16acb60152c254244a6e3edcd5e66c029caf909f4f2dc4972ff01716d96de84afc62d94c47810b55b38eb0c3572b728552611d3fbb0209e100fde2e8404d168dd431c6d3dd2eec1574d86ad34e2374506cb98ee4576bbf5a78f07003c653d9e8887582020fb369c16fe1ae5e46f0dd4e0b00771fb7ab745c15389be291530859010281f2746b652118a47b07b9763c789e90576f6a2585d530a89892aa3aba2577895e63a3492cbbf38d00", + "data": "~Ǹǘ肄㮚쨀艒ࠃ隫⢖즯際ᄹ丿௧僚痐ݮᴚﶈ~čǠ뚳Ꝥ~ĀĀ肸ꬖ겶Œ쉔⑊渾~ǜǕ~ǦŬʜ꾐齏ⷄ霯~ǰėᛙ淨䫼拙䱇脋喳躰썗⭲蕒愝㾻ȉ~ǡĀ~ǽǢ~Ǩŀ䴖跔㇆폝⻬ᕴ~ǘŪ퍎⍴偬릎~Ǥŗ殿婸~ǰŰφ叙~Ǩƈ疂ȏ덩셯~ǡƮ幆~ǰǝ下wᾷꭴ尕㢛~ǢƑ匈夁ʁ~DzŴ步℘ꑻ~ćƹ瘼碞遗潪▅픰ꢘ銪㪺╷襞掣䤬믳贀", + "description": "Correct transaction on Devin.", + "length": 145, + "sms": 3, + "mms": 1 + }, + { + "hex": "f8dc821ae4850ee6b280008252080196cb65d677385703c528527f2a0f0e401b4af1988d91c5896e3f4f2ab21845000080b8abcffa127f34f8dc8d8bc9a50da5def786a16ecab58d9d1cdc3e1347077f531ad0339797568345464f542f8da3bcd50fd683878f52e6d094610025d6e4a5fb3699acd20ebd1ec2fdde9d12f5e82fe5f4c8d9061466475b3293bb18c34504c6eb43bc0ba48d61a8edfda686c69773fa96b90d00760d8277330d90589ba26fb63874952b013a8af1a5edacbcabb37108b47518c79abd6e50be00da0a08fb9126fd265175cace1ac93d1f809b80", + "data": "~Ǹǜ舚~Ǥƅ~ĎǦ늀~ĀƂ刈Ɩ쭥홷㡗υ⡒缪༎䀛䫱颍釅襮㽏⪲ᡅ~ĀĀ肸ꯏ晴缴~Ǹǜ趋즥ඥ~ǞǷ蚡滊떍鴜~ǜľፇݿ匚퀳鞗嚃䕆佔⾍ꎼ픏횃螏勦킔愀◖~Ǥƥזּ馬툎봞싽~ǞƝድ~Ǩį~ǥǴ죙ؔ晇嬲鎻ᣃ䔄웫䎼த赡꣭ﶦ蛆靳殺뤍vං眳ඐ墛ꉯ똸璕⬁㪊~DZƥ~ǭƬ벫덱ࢴ甘잚뵮傾Úਈﮑ⛽♑痊츚줽ᾀ鮀", + "description": "Correct transaction without 0x prefix.", + "length": 139, + "sms": 3, + "mms": 1 + } + ], + "invalid": [ + { + "hex": "0xKKKK80843b9aca008252080396ab2896c9af969b11394e3f0be750da75d0076e1d1afd880de0b6b3a764000080b8ab16acb60152c254244a6e3edcd5e66c029caf909f4f2dc4972ff01716d96de84afc62d94c47810b55b38eb0c3572b728552611d3fbb0209e100fde2e8404d168dd431c6d3dd2eec1574d86ad34e2374506cb98ee4576bbf5a78f07003c653d9e8887582020fb369c16fe1ae5e46f0dd4e0b00771fb7ab745c15389be291530859010281f2746b652118a47b07b9763c789e90576f6a2585d530a89892aa3aba2577895e63a3492cbbf38d00", + "description": "Not a hex string.", + "code": "415" + } + ] +} diff --git a/tools/check_conan_center.py b/tools/check_conan_center.py new file mode 100644 index 0000000..2aa1db5 --- /dev/null +++ b/tools/check_conan_center.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 +"""Exercise the staged download recipe against the archive before publication.""" +import argparse +from functools import partial +import http.server +from pathlib import Path +import shutil +import subprocess +import tempfile +import threading +import yaml + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--recipe", required=True) + parser.add_argument("--archive", required=True) + parser.add_argument("--version", required=True) + args = parser.parse_args() + archive = Path(args.archive).resolve() + handler = partial(http.server.SimpleHTTPRequestHandler, directory=str(archive.parent)) + server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), handler) + threading.Thread(target=server.serve_forever, daemon=True).start() + try: + with tempfile.TemporaryDirectory(prefix="txms-conan-recipe-") as folder: + shutil.copytree(args.recipe, folder, dirs_exist_ok=True) + data_path = Path(folder, "conandata.yml") + data = yaml.safe_load(data_path.read_text()) + data["sources"][args.version]["url"] = f"http://127.0.0.1:{server.server_port}/{archive.name}" + data_path.write_text(yaml.safe_dump(data)) + for shared in ("False", "True"): + subprocess.run(["conan", "create", folder, "--version", args.version, "--build=missing", "--no-remote", "-o", f"libtxms/*:shared={shared}"], check=True) + finally: + server.shutdown() + server.server_close() diff --git a/tools/prepare_conan_center.py b/tools/prepare_conan_center.py new file mode 100644 index 0000000..0cdf2f5 --- /dev/null +++ b/tools/prepare_conan_center.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 +"""Stage a ConanCenter recipe with an exact release archive checksum.""" +import argparse +import hashlib +from pathlib import Path +import re +import shutil +import yaml +from release import VERSION + +def prepare(root, version, archive, output): + root, archive, output = Path(root), Path(archive), Path(output) + if not re.fullmatch(VERSION, version): + raise ValueError("Expected a stable MAJOR.MINOR.PATCH version") + if archive.name != f"libtxms-{version}.tar.gz": + raise ValueError("Archive filename does not match the version") + recipe = output / "recipes" / "libtxms" + all_dir = recipe / "all" + all_dir.mkdir(parents=True, exist_ok=True) + shutil.copy2(root / "packaging/LICENSE", output / "LICENSE") + shutil.copy2(root / "packaging/conan-center/conanfile.py", all_dir / "conanfile.py") + shutil.copytree(root / "test_package", all_dir / "test_package", dirs_exist_ok=True, + ignore=shutil.ignore_patterns("build*", "__pycache__", "CMakeUserPresets.json")) + data = {"sources": {version: { + "url": f"https://github.com/DataLayerHost/libtxms/releases/download/{version}/{archive.name}", + "sha256": hashlib.sha256(archive.read_bytes()).hexdigest(), + }}} + (all_dir / "conandata.yml").write_text(yaml.safe_dump(data, sort_keys=False)) + (recipe / "config.yml").write_text(yaml.safe_dump({"versions": {version: {"folder": "all"}}}, sort_keys=False)) + return recipe + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--version", required=True) + parser.add_argument("--archive", required=True) + parser.add_argument("--output", default="release-stage") + parser.add_argument("--root", default=".") + args = parser.parse_args() + print(prepare(args.root, args.version, args.archive, args.output)) diff --git a/tools/release.py b/tools/release.py new file mode 100644 index 0000000..44444b5 --- /dev/null +++ b/tools/release.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +"""Create checksummed source assets from the exact tagged commit; never publish.""" +import argparse +import hashlib +import json +from pathlib import Path +import re +import subprocess + +VERSION = r"(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)" + +def build_release(root, project, tag, output): + root, output = Path(root).resolve(), Path(output).resolve() + if project not in ("libtxms", "kamailio-txms") or not re.fullmatch(VERSION, tag): + raise ValueError("Use a stable MAJOR.MINOR.PATCH tag and a known project") + def git(*args): + return subprocess.check_output(["git", "-C", str(root), *args], text=True).strip() + commit = git("rev-parse", "HEAD") + if git("rev-parse", f"refs/tags/{tag}^{{commit}}") != commit: + raise ValueError("The release tag must point to the checked-out commit") + cmake = git("show", f"{commit}:CMakeLists.txt") + match = re.search(r"project\(\w+ VERSION (" + VERSION + r")\b", cmake) + if not match or match.group(1) != tag: + raise ValueError("Tag and CMake project version differ") + if git("status", "--porcelain", "--untracked-files=no"): + raise ValueError("Tracked files are dirty; commit changes before releasing") + output.mkdir(parents=True, exist_ok=True) + archive = output / f"{project}-{tag}.tar.gz" + subprocess.run(["git", "-C", str(root), "archive", "--format=tar.gz", f"--prefix={project}-{tag}/", f"--output={archive}", commit], check=True) + digest = hashlib.sha256(archive.read_bytes()).hexdigest() + metadata = {"project": project, "version": tag, "tag": tag, "commit": commit, "source_sha256": digest} + if project == "kamailio-txms": + dependency = git("show", f"{commit}:LIBTXMS_VERSION") + if not re.fullmatch(VERSION, dependency): + raise ValueError("Invalid LIBTXMS_VERSION") + metadata["libtxms_tag"] = dependency + (output / "release.json").write_text(json.dumps(metadata, indent=2) + "\n") + (output / "SHA256SUMS").write_text(f"{digest} {archive.name}\n") + return metadata + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--project", required=True) + parser.add_argument("--tag", required=True) + parser.add_argument("--root", default=".") + parser.add_argument("--output", default="dist") + args = parser.parse_args() + print(json.dumps(build_release(args.root, args.project, args.tag, args.output), indent=2)) diff --git a/tools/submit_conan_center.py b/tools/submit_conan_center.py new file mode 100644 index 0000000..f7f909f --- /dev/null +++ b/tools/submit_conan_center.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 +"""Open an idempotent recipe PR using an explicitly configured ConanCenter fork.""" +import argparse +import json +import os +from pathlib import Path +import re +import shutil +import subprocess +import tempfile +import yaml +from release import VERSION + +UPSTREAM = "conan-io/conan-center-index" + + +def merge_recipe(stage, destination, version): + """Preserve older versions and refuse to replace an existing source checksum.""" + source = Path(stage) / "recipes/libtxms" + target = Path(destination) / "recipes/libtxms" + data = yaml.safe_load((source / "all/conandata.yml").read_text()) + new_source = data["sources"][version] + previous_path = target / "all/conandata.yml" + previous = yaml.safe_load(previous_path.read_text()) if previous_path.exists() else {"sources": {}} + old_source = previous.setdefault("sources", {}).get(version) + if old_source is not None and old_source != new_source: + raise ValueError("Existing version has different source metadata; never overwrite a published version") + config_path = target / "config.yml" + config = yaml.safe_load(config_path.read_text()) if config_path.exists() else {"versions": {}} + if version in config.get("versions", {}) and config["versions"][version].get("folder") != "all": + raise ValueError("Existing version uses another recipe folder; manual review is required") + previous["sources"][version] = new_source + config.setdefault("versions", {})[version] = {"folder": "all"} + (target / "all").mkdir(parents=True, exist_ok=True) + shutil.copy2(source / "all/conanfile.py", target / "all/conanfile.py") + shutil.copytree(source / "all/test_package", target / "all/test_package", dirs_exist_ok=True) + previous_path.write_text(yaml.safe_dump(previous, sort_keys=False)) + config_path.write_text(yaml.safe_dump(config, sort_keys=False)) + + +def submit(fork, version, stage): + if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9-]*/[A-Za-z0-9_.-]+", fork) or fork.lower() == UPSTREAM.lower(): + raise ValueError("Set CONAN_CENTER_FORK to your own owner/repository fork") + if not re.fullmatch(VERSION, version): + raise ValueError("Invalid stable version") + if not os.environ.get("GH_TOKEN"): + raise ValueError("CONAN_CENTER_TOKEN is required to submit to the configured fork") + def run(*args, cwd=None): + return subprocess.check_output(args, cwd=cwd, text=True).strip() + info = json.loads(run("gh", "api", f"repos/{fork}")) + if info.get("parent", {}).get("full_name") != UPSTREAM: + raise ValueError("Configured repository is not a fork of ConanCenterIndex") + base = run("gh", "api", f"repos/{UPSTREAM}", "--jq", ".default_branch") + branch = f"automation/libtxms-{version}" + owner = fork.split("/")[0] + with tempfile.TemporaryDirectory(prefix="txms-cci-pr-") as folder: + repo = Path(folder, "index") + # gh auth configures a credential helper; the token never appears in URLs. + run("gh", "auth", "setup-git") + run("git", "clone", "--filter=blob:none", "--sparse", f"https://github.com/{fork}.git", str(repo)) + run("git", "sparse-checkout", "set", "recipes/libtxms", cwd=repo) + run("git", "remote", "add", "upstream", f"https://github.com/{UPSTREAM}.git", cwd=repo) + run("git", "fetch", "upstream", base, cwd=repo) + existing = run("git", "ls-remote", "--heads", "origin", f"refs/heads/{branch}", cwd=repo) + if existing: + run("git", "fetch", "origin", branch, cwd=repo) + run("git", "checkout", "-b", branch, "FETCH_HEAD", cwd=repo) + else: + run("git", "checkout", "-b", branch, f"upstream/{base}", cwd=repo) + merge_recipe(stage, repo, version) + run("git", "add", "--", "recipes/libtxms", cwd=repo) + if run("git", "diff", "--cached", "--name-only", cwd=repo): + run("git", "-c", "user.name=TxMS release automation", "-c", "user.email=41898282+github-actions[bot]@users.noreply.github.com", "commit", "-m", f"libtxms: add version {version}", cwd=repo) + run("git", "push", "origin", f"HEAD:refs/heads/{branch}", cwd=repo) + elif not existing: + print("ConanCenter already contains this recipe; no PR needed") + return + # REST supports organization-owned forks and owner-qualified heads. + # gh pr create/list do not support all of these combinations. + if not run("git", "diff", f"upstream/{base}", "HEAD", "--", "recipes/libtxms", cwd=repo): + print("Recipe changes are already upstream; no PR needed") + return + prs = json.loads(run("gh", "api", "--method", "GET", f"repos/{UPSTREAM}/pulls", "-f", f"head={owner}:{branch}", "-f", "state=open")) + if prs: + print(prs[0]["html_url"]) + return + body = Path(folder, "body.md") + body.write_text(f"Adds libtxms {version}, the standalone C11 TxMS codec.\n\n" + f"Source release: https://github.com/DataLayerHost/libtxms/releases/tag/{version}\n\n" + "The recipe downloads the release archive with its SHA-256 checksum. Static and shared packages were built and their C consumer was executed before submission.\n\n" + "Library license: LicenseRef-libtxms-CORE; the complete CORE license and NOTICE are packaged. Recipe and test_package are MIT licensed. Please review custom-license eligibility.\n") + request = Path(folder, "pull-request.json") + request.write_text(json.dumps({"base": base, "head": f"{owner}:{branch}", "head_repo": fork.split("/")[1], + "title": f"libtxms/{version}: add recipe", "body": body.read_text()})) + print(run("gh", "api", "--method", "POST", f"repos/{UPSTREAM}/pulls", "--input", str(request), "--jq", ".html_url")) + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--fork", required=True) + parser.add_argument("--version", required=True) + parser.add_argument("--stage", default="release-stage") + args = parser.parse_args() + submit(args.fork, args.version, Path(args.stage).resolve()) diff --git a/tools/unicode.mjs b/tools/unicode.mjs new file mode 100644 index 0000000..8cdd93a --- /dev/null +++ b/tools/unicode.mjs @@ -0,0 +1,11 @@ +// Regenerate with the same Node Unicode version used by the behavioral oracle. +const ranges = []; +for (let c = 0; c < 65536; c++) { + if (!/[\p{C}\p{Z}]/u.test(String.fromCharCode(c))) continue; + if (ranges.length && ranges.at(-1)[1] === c - 1) ranges.at(-1)[1] = c; + else ranges.push([c, c]); +} +console.log(`/* Generated from Node ${process.version}, Unicode ${process.versions.unicode}. */`); +console.log('static const uint16_t txms_ranges[][2] = {'); +for (const [a,b] of ranges) console.log(`\t{0x${a.toString(16)}, 0x${b.toString(16)}},`); +console.log('};'); diff --git a/tools/verify_release.py b/tools/verify_release.py new file mode 100644 index 0000000..9def980 --- /dev/null +++ b/tools/verify_release.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 +"""Verify downloaded release sources and metadata against the checked-out tag.""" +import argparse +import hashlib +import json +from pathlib import Path +import re +import subprocess +from release import VERSION + + +def verify(directory, tag, commit): + if not re.fullmatch(VERSION, tag): + raise ValueError("Invalid stable release tag") + directory = Path(directory) + archive_name = f"libtxms-{tag}.tar.gz" + checksums = {} + for line in (directory / "SHA256SUMS").read_text().splitlines(): + match = re.fullmatch(r"([0-9a-f]{64}) ([A-Za-z0-9_.-]+)", line) + if not match or match.group(2) in checksums: + raise ValueError("Malformed or duplicate checksum entry") + checksums[match.group(2)] = match.group(1) + for name in (archive_name, "release.json"): + if checksums.get(name) != hashlib.sha256((directory / name).read_bytes()).hexdigest(): + raise ValueError(f"Checksum mismatch for {name}") + metadata = json.loads((directory / "release.json").read_text()) + if metadata != {"project": "libtxms", "version": tag, "tag": tag, "commit": commit, "source_sha256": checksums[archive_name]}: + raise ValueError("Release metadata does not match the checked-out tag") + return metadata + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--tag", required=True) + parser.add_argument("--directory", default="dist") + args = parser.parse_args() + commit = subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip() + tag_commit = subprocess.check_output(["git", "rev-parse", f"refs/tags/{args.tag}^{{commit}}"], text=True).strip() + if commit != tag_commit: + raise ValueError("Checkout differs from release tag") + print(json.dumps(verify(args.directory, args.tag, commit), indent=2)) From 3bfa694ba8c23d4f3de49c1cafc28f4c1ff8ebfd Mon Sep 17 00:00:00 2001 From: Rastislav Date: Mon, 28 Sep 2026 14:18:34 +0200 Subject: [PATCH 2/2] fix --- .github/workflows/release.yml | 15 +++++++------ README.md | 36 +++++++++++++----------------- tests/test_release.py | 42 +++++++++++++++++++++++++++++++++-- tools/release.py | 6 ++++- 4 files changed, 68 insertions(+), 31 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 18fc0e9..bcf3034 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,7 +1,7 @@ name: Release on: - push: - tags: ['[0-9]*.[0-9]*.[0-9]*'] + release: + types: [published] permissions: contents: read concurrency: @@ -9,12 +9,13 @@ concurrency: cancel-in-progress: false jobs: tests: + if: ${{ !github.event.release.prerelease }} uses: ./.github/workflows/ci.yml assets: needs: tests runs-on: ubuntu-latest env: - TAG: ${{ github.ref_name }} + TAG: ${{ github.event.release.tag_name }} steps: - uses: actions/checkout@v4 with: @@ -50,7 +51,7 @@ jobs: env: GH_TOKEN: ${{ github.token }} GH_REPO: ${{ github.repository }} - TAG: ${{ github.ref_name }} + TAG: ${{ github.event.release.tag_name }} steps: - uses: actions/download-artifact@v4 with: @@ -58,13 +59,13 @@ jobs: path: dist - name: Verify downloaded assets run: cd dist && sha256sum --check SHA256SUMS - - name: Publish the tested release - run: gh release create "$TAG" dist/* --verify-tag --title "$TAG" --generate-notes + - name: Attach tested assets to the published release + run: gh release upload "$TAG" dist/* conan-center: needs: publish if: vars.CONAN_CENTER_FORK != '' uses: ./.github/workflows/conan-center.yml with: - tag: ${{ github.ref_name }} + tag: ${{ github.event.release.tag_name }} secrets: CONAN_CENTER_TOKEN: ${{ secrets.CONAN_CENTER_TOKEN }} diff --git a/README.md b/README.md index 93afad3..31cb03c 100644 --- a/README.md +++ b/README.md @@ -94,31 +94,25 @@ run in hosted CI and have not been executed locally. ## GitHub releases -[release.yml](.github/workflows/release.yml) runs on stable `MAJOR.MINOR.PATCH` -tags. It calls the complete CI workflow at the same tagged commit, including -sanitizers, compatibility tests, fuzz checks and Conan consumer tests. Publishing -is blocked until every CI job and release-package check succeeds. - -1. Update the version in `CMakeLists.txt`, commit the release changes and push them. -2. Create and push the matching tag, for example: - - ```sh - git tag -a 0.1.0 -m 'libtxms 0.1.0' - git push origin 0.1.0 - ``` - -3. The workflow verifies that the tag matches the CMake version and publishes: +[release.yml](.github/workflows/release.yml) starts when a stable GitHub Release +is **published**, using tags such as `0.1.0` without a `v` prefix. It calls the +complete CI workflow at the tagged commit, including sanitizers, compatibility +tests, fuzz checks and Conan consumer tests. The release is already visible while +tests run; assets are attached only after all tests and package checks pass. + +1. Update `CMakeLists.txt`, commit the version/workflow changes and push them. +2. Create and publish a GitHub Release for the matching tag, such as `0.1.0`, + through GitHub's Releases page. Pushing a tag alone does not run this workflow. +3. After checking the tag/version and passing tests, the workflow uploads: - `libtxms-0.1.0.tar.gz`: source from the exact tested Git commit. - `libtxms-0.1.0-conan-center.tar.gz`: staged ConanCenter recipe and consumer test. - `release.json`: version, commit and source checksum. - `SHA256SUMS`: checksums for all release assets. -The workflow first builds static/shared ConanCenter packages from the same -archive served locally to exercise download and checksum verification. It then -publishes a GitHub Release with generated notes using the repository's normal -`GITHUB_TOKEN`. Existing releases are never overwritten; rerun a failed job -before publication, or issue a new version after publication. No prebuilt -platform libraries are attached; Conan builds the appropriate package. +The workflow tests static/shared ConanCenter packages from that same archive, +then uploads assets to the existing release using `GITHUB_TOKEN`. Prereleases +are skipped. Existing assets are not overwritten. No prebuilt platform libraries +are attached; Conan builds the appropriate package. ## ConanCenter submission automation @@ -141,7 +135,7 @@ the token in source files. A first-time submitter may need to sign ConanCenter's CLA. ConanCenter maintainers also decide acceptance of the custom CORE license; the recipe declares `LicenseRef-libtxms-CORE` and includes its complete text. -When the fork variable is set, successful release publication directly calls +When the fork variable is set, successful release asset upload directly calls [conan-center.yml](.github/workflows/conan-center.yml). This avoids relying on a second release event generated by `GITHUB_TOKEN`. The workflow downloads the published source, checks its checksum and commit metadata, rebuilds static/shared diff --git a/tests/test_release.py b/tests/test_release.py index 0b018b6..b223440 100644 --- a/tests/test_release.py +++ b/tests/test_release.py @@ -6,6 +6,7 @@ import tarfile import tempfile import unittest +import yaml ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / "tools")) @@ -22,6 +23,7 @@ def setUp(self): (self.root / "CMakeLists.txt").write_text("project(txms VERSION 0.1.0 LANGUAGES C)\n") (self.root / "LICENSE").write_text("Test fixture license\n") (self.root / "LIBTXMS_VERSION").write_text("0.1.0\n") + (self.root / "LIBTXMS_REVISION").write_text("a" * 40 + "\n") self.git("add", ".") self.git("commit", "-qm", "fixture") self.git("tag", "0.1.0") @@ -45,7 +47,8 @@ def test_archive_is_tagged_tree_with_checksums(self): def test_gateway_records_tested_dependency(self): metadata = build_release(self.root, "kamailio-txms", "0.1.0", self.root / "dist") - self.assertEqual(metadata["libtxms_tag"], "0.1.0") + self.assertEqual(metadata["libtxms_version"], "0.1.0") + self.assertEqual(metadata["libtxms_commit"], "a" * 40) def test_mismatched_or_unsafe_version_rejected(self): self.git("tag", "0.2.0") @@ -65,8 +68,43 @@ def test_checkout_must_match_tag(self): build_release(self.root, "libtxms", "0.1.0", self.root / "dist") + def test_invalid_dependency_revision_rejected(self): + (self.root / "LIBTXMS_REVISION").write_text("main\n") + self.git("commit", "-qam", "invalid dependency revision") + self.git("tag", "-f", "0.1.0") + with self.assertRaises(ValueError): + build_release(self.root, "kamailio-txms", "0.1.0", self.root / "dist") + + +class WorkflowTests(unittest.TestCase): + def test_published_release_uploads_only_after_tests(self): + workflow = yaml.safe_load((ROOT / ".github/workflows/release.yml").read_text()) + # YAML 1.1 parses the Actions `on` key as True. + self.assertEqual(workflow[True], {"release": {"types": ["published"]}}) + jobs = workflow["jobs"] + self.assertEqual(jobs["tests"]["uses"], "./.github/workflows/ci.yml") + self.assertEqual(jobs["assets"]["needs"], "tests") + self.assertEqual(set(jobs["publish"]["needs"]), {"tests", "assets"}) + self.assertEqual(jobs["publish"]["env"]["TAG"], "${{ github.event.release.tag_name }}") + commands = "\n".join(step.get("run", "") for step in jobs["publish"]["steps"]) + self.assertIn('gh release upload "$TAG"', commands) + self.assertNotIn("gh release create", commands) + if "conan-center" in jobs: + self.assertEqual(jobs["conan-center"]["needs"], "publish") + self.assertEqual(jobs["conan-center"]["with"]["tag"], "${{ github.event.release.tag_name }}") + + @unittest.skipUnless((ROOT / "LIBTXMS_REVISION").exists(), "Gateway dependency check") + def test_gateway_checks_out_pinned_commit_in_every_build(self): + self.assertRegex((ROOT / "LIBTXMS_REVISION").read_text().strip(), r"^[0-9a-f]{40}$") + workflow = yaml.safe_load((ROOT / ".github/workflows/ci.yml").read_text()) + for name in ("core", "kamailio", "fuzz"): + steps = workflow["jobs"][name]["steps"] + checkout = next(step for step in steps if step.get("with", {}).get("repository") == "DataLayerHost/libtxms") + self.assertEqual(checkout["with"]["ref"], "${{ env.LIBTXMS_REF }}") + self.assertTrue(any("cat LIBTXMS_REVISION" in step.get("run", "") for step in steps)) + + if (ROOT / "tools/prepare_conan_center.py").exists(): - import yaml from prepare_conan_center import prepare from submit_conan_center import merge_recipe from verify_release import verify diff --git a/tools/release.py b/tools/release.py index 44444b5..c15bda7 100644 --- a/tools/release.py +++ b/tools/release.py @@ -33,7 +33,11 @@ def git(*args): dependency = git("show", f"{commit}:LIBTXMS_VERSION") if not re.fullmatch(VERSION, dependency): raise ValueError("Invalid LIBTXMS_VERSION") - metadata["libtxms_tag"] = dependency + revision = git("show", f"{commit}:LIBTXMS_REVISION") + if not re.fullmatch(r"[0-9a-f]{40}", revision): + raise ValueError("Invalid LIBTXMS_REVISION: expected a full commit SHA") + metadata["libtxms_version"] = dependency + metadata["libtxms_commit"] = revision (output / "release.json").write_text(json.dumps(metadata, indent=2) + "\n") (output / "SHA256SUMS").write_text(f"{digest} {archive.name}\n") return metadata