diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..a409261 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,75 @@ +name: "CI" + +on: + push: + branches: + - main + + pull_request: + +jobs: + build: + runs-on: 'ubuntu-latest' + strategy: + matrix: + include: + - id: gcc-12 + cc: gcc-12 + cxx: g++-12 + - id: clang + cc: clang + cxx: clang++ + name: Build (${{ matrix.cc }}) + env: + CC: ${{ matrix.cc }} + CXX: ${{ matrix.cxx }} + steps: + - uses: actions/checkout@v4 + - name: CMake + run: | + sudo apt-get update + sudo apt-get install libhdf5-dev gcc-12 g++-12 clang + cmake -B build + - name: Build + run: VERBOSE=true make -C build -j `nproc` + - name: Archive Build + run: tar -czf build.tar.gz build + - uses: actions/upload-artifact@v4 + with: + name: compliance-build-${{ matrix.id }} + path: build.tar.gz + retention-days: 1 + + test: + name: Test (${{ matrix.cc }}) + needs: build + runs-on: ubuntu-latest + timeout-minutes: 60 + strategy: + matrix: + include: + - id: gcc-12 + cc: gcc-12 + - id: clang + cc: clang + steps: + - uses: actions/checkout@v4 + - uses: actions/download-artifact@v4 + with: + name: compliance-build-${{ matrix.id }} + - name: Restore Build + run: tar -xzf build.tar.gz + - name: Install Runtime Dependencies + run: | + sudo apt-get update + sudo apt-get install libhdf5-dev + - uses: prefix-dev/setup-pixi@v0.10.0 + with: + run-install: false + cache: false + - name: CMake Tests + run: ctest --test-dir build --output-on-failure + - name: Compliance Tests + env: + BINSPARSE_BUILD_DIR: ${{ github.workspace }}/build + run: ./test/compliance/run-binsparse-tests.sh -q diff --git a/.gitignore b/.gitignore index fffab4b..90a93e7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ scripts venv build +build-* ._* tensor_test_files diff --git a/CMakeLists.txt b/CMakeLists.txt index 54ac02f..120fd1a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,6 +1,6 @@ include(FetchContent) -cmake_minimum_required(VERSION 3.5) +cmake_minimum_required(VERSION 3.20) project(bsp-to-taco) add_library(bsp-to-taco SHARED) @@ -8,8 +8,8 @@ add_library(bsp-to-taco SHARED) cmake_policy(SET CMP0079 NEW) set(CMAKE_C_STANDARD 11) - -#set(CMAKE_CXX_STANDARD 20) +set(CMAKE_POSITION_INDEPENDENT_CODE ON) +target_compile_features(bsp-to-taco PUBLIC cxx_std_11) #set(CMAKE_C_FLAGS "-O3 -march=native") set(CMAKE_CXX_FLAGS "-O3 -march=native -Wall -Wno-narrowing -fPIC") @@ -21,8 +21,8 @@ add_subdirectory(src) FetchContent_Declare( binsparse - GIT_REPOSITORY https://github.com/junikimm717/binsparse-reference-c - GIT_TAG main + GIT_REPOSITORY https://github.com/Binsparse/binsparse-reference-c + GIT_TAG 13e56902c6f729822730297a5d13bbdbc62e619b ) FetchContent_MakeAvailable(binsparse) target_include_directories(bsp-to-taco PUBLIC ${binsparse_SOURCE_DIR}/include) @@ -65,7 +65,7 @@ target_include_directories(${PROJECT_NAME} PUBLIC ${taco_SOURCE_DIR}/include) #### Project Configuration target_include_directories(${PROJECT_NAME} PUBLIC ${CMAKE_BINARY_DIR}/include) -target_link_libraries(${PROJECT_NAME} PUBLIC binsparse-rc taco) +target_link_libraries(${PROJECT_NAME} PUBLIC binsparse taco) if(CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME) add_subdirectory(examples) diff --git a/include/bsp_taco/bsp_to_taco.hpp b/include/bsp_taco/bsp_to_taco.hpp index 9680ade..16b94c8 100644 --- a/include/bsp_taco/bsp_to_taco.hpp +++ b/include/bsp_taco/bsp_to_taco.hpp @@ -1,11 +1,12 @@ -#include #include +#include #include #pragma once namespace bsp_taco { -taco::TensorBase makeTacoTensor(bsp_tensor_t& tensor); +taco::TensorBase makeTacoTensor(bsp_tensor_t& tensor, + const bsp_array_t* fill = nullptr); taco::TensorBase readBinSparse(std::string filename); } diff --git a/include/bsp_taco/taco_to_bsp.hpp b/include/bsp_taco/taco_to_bsp.hpp index 5fcd88c..c6dccdf 100644 --- a/include/bsp_taco/taco_to_bsp.hpp +++ b/include/bsp_taco/taco_to_bsp.hpp @@ -1,4 +1,3 @@ -#include "taco/tensor.h" #include #include #include diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 9834ff4..6cd8c3c 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -3,6 +3,6 @@ # SPDX-License-Identifier: BSD-3-Clause target_sources(bsp-to-taco PRIVATE - src/bsp_to_taco.cpp - src/taco_to_bsp.cpp + bsp_to_taco.cpp + taco_to_bsp.cpp ) diff --git a/src/bsp_to_taco.cpp b/src/bsp_to_taco.cpp index bfa24f0..ff4ab0d 100644 --- a/src/bsp_to_taco.cpp +++ b/src/bsp_to_taco.cpp @@ -1,5 +1,9 @@ #include +#include +#include +#include + static std::vector getDimensions(bsp_tensor_t& tensor) { std::vector dims(tensor.rank); for (int i = 0; i < tensor.rank; i++) { @@ -45,20 +49,63 @@ static inline taco::Datatype getTacoDataType(bsp_array_t& array) { static taco::Array bspToTacoArray(bsp_array_t& array) { taco::Datatype dataType = getTacoDataType(array); - bsp_type_t type = array.type; taco::Array res = taco::makeArray(dataType, array.size); // eventually, get rid of memcpy? memcpy(res.getData(), array.data, array.size * dataType.getNumBytes()); return res; } +static taco::Array bspIndexArrayToTacoInt32Array(bsp_array_t& array) { + taco::Array res = taco::makeArray(taco::Int32, array.size); + auto* out = static_cast(res.getData()); + for (size_t i = 0; i < array.size; ++i) { + size_t value = 0; + switch (array.type) { + case BSP_UINT8: + value = static_cast(array.data)[i]; + break; + case BSP_UINT16: + value = static_cast(array.data)[i]; + break; + case BSP_UINT32: + value = static_cast(array.data)[i]; + break; + case BSP_UINT64: + value = static_cast(array.data)[i]; + break; + case BSP_INT8: + case BSP_BINT8: + value = static_cast(array.data)[i]; + break; + case BSP_INT16: + value = static_cast(array.data)[i]; + break; + case BSP_INT32: + value = static_cast(array.data)[i]; + break; + case BSP_INT64: + value = static_cast(array.data)[i]; + break; + default: + taco_uerror << "Unsupported Binsparse coordinate type"; + } + taco_uassert(value <= static_cast(INT32_MAX)) + << "Binsparse coordinate is too large for TACO int32 storage"; + out[i] = static_cast(value); + } + return res; +} + static taco::Format createTacoFormat(bsp_tensor_t& tensor) { bsp_level_t* level = tensor.level; std::vector modeTypes; while (level->kind != BSP_TENSOR_ELEMENT) { switch (level->kind) { case BSP_TENSOR_DENSE: { - modeTypes.push_back(taco::Dense()); + bsp_dense_t* data = (bsp_dense_t*) level->data; + for (int i = 0; i < data->rank; i++) { + modeTypes.push_back(taco::Dense()); + } level = ((bsp_dense_t*) level->data)->child; break; } @@ -86,68 +133,105 @@ static taco::Format createTacoFormat(bsp_tensor_t& tensor) { return taco::Format(modeTypes); } +static size_t storedDimension(bsp_tensor_t& tensor, int depth) { + size_t logical = tensor.transpose != NULL ? tensor.transpose[depth] : depth; + taco_uassert(logical < (size_t) tensor.rank) + << "Binsparse transpose is not a permutation"; + return tensor.dims[logical]; +} + static taco::Index createTacoIndex(bsp_tensor_t& tensor, taco::Format& format) { bsp_level_t* level = tensor.level; - size_t* dims = tensor.dims; std::vector modeIndices; + int depth = 0; while (level->kind != BSP_TENSOR_ELEMENT) { switch (level->kind) { case BSP_TENSOR_DENSE: { - modeIndices.push_back(taco::ModeIndex({taco::makeArray({(int) *dims})})); + bsp_dense_t* data = (bsp_dense_t*) level->data; + for (int i = 0; i < data->rank; i++) { + size_t dimension = storedDimension(tensor, depth + i); + taco_uassert(dimension <= static_cast(INT32_MAX)) + << "Binsparse dimension is too large for TACO int32 storage"; + modeIndices.push_back( + taco::ModeIndex({taco::makeArray({(int) dimension})})); + } + depth += data->rank; level = ((bsp_dense_t*) level->data)->child; break; } case BSP_TENSOR_SPARSE: { bsp_sparse_t* data = (bsp_sparse_t*) level->data; - if (data->pointers_to != NULL && data->pointers_to->type != BSP_INT32) { - taco_uerror << "pointers_to just be an int32 type to interface " - << "properly with taco!"; - } modeIndices.push_back(taco::ModeIndex({ data->pointers_to != NULL - ? taco::Array(getTacoDataType(*data->pointers_to), - data->pointers_to->data, data->pointers_to->size, - taco::Array::Free) + ? bspIndexArrayToTacoInt32Array(*data->pointers_to) : taco::makeArray({ (int) 0, (int) data->indices[0].size, }), - taco::Array(getTacoDataType(data->indices[0]), data->indices[0].data, - data->indices[0].size, taco::Array::Free), + bspIndexArrayToTacoInt32Array(data->indices[0]), })); for (int i = 1; i < data->rank; i++) { - if (data->indices[i].type != BSP_INT32) - taco_terror << "indices just be an int32 type to interface properly " - << "with taco!"; modeIndices.push_back(taco::ModeIndex({ - taco::makeArray(getTacoDataType(data->indices[i]), 0), - taco::Array(getTacoDataType(data->indices[i]), - data->indices[i].data, data->indices[i].size, - taco::Array::Free), + taco::makeArray(taco::Int32, 0), + bspIndexArrayToTacoInt32Array(data->indices[i]), })); } + depth += data->rank; level = ((bsp_sparse_t*) level->data)->child; break; } default:; } - dims++; } return taco::Index(format, modeIndices); } +template +static taco::Literal scalarLiteral(const bsp_array_t& array) { + T value; + memcpy(&value, array.data, sizeof(value)); + return taco::Literal(value); +} + +static taco::Literal getFillLiteral(const bsp_array_t& fill) { + switch (fill.type) { + case BSP_UINT8: return scalarLiteral(fill); + case BSP_UINT16: return scalarLiteral(fill); + case BSP_UINT32: return scalarLiteral(fill); + case BSP_UINT64: return scalarLiteral(fill); + case BSP_INT8: + case BSP_BINT8: return scalarLiteral(fill); + case BSP_INT16: return scalarLiteral(fill); + case BSP_INT32: return scalarLiteral(fill); + case BSP_INT64: return scalarLiteral(fill); + case BSP_FLOAT32: return scalarLiteral(fill); + case BSP_FLOAT64: return scalarLiteral(fill); + case BSP_COMPLEX_FLOAT32: + return scalarLiteral>(fill); + case BSP_COMPLEX_FLOAT64: + return scalarLiteral>(fill); + default: + taco_uerror << "Unsupported fill type supplied to taco converter"; + return taco::Literal(); + } +} + /* Creates a taco object from a bsp tensor. Note that this function **consumes** the bsp tensor object! */ -taco::TensorBase bsp_taco::makeTacoTensor(bsp_tensor_t& tensor) { - bsp_level_t* level = tensor.level; +taco::TensorBase bsp_taco::makeTacoTensor(bsp_tensor_t& tensor, + const bsp_array_t* fill) { + if (tensor.is_iso) + taco_uerror << "TACO does not support iso-valued tensor storage"; bsp_array_t values = bsp_get_tensor_values(tensor); taco::Format tacoFormat = createTacoFormat(tensor); taco::Index tacoIndex = createTacoIndex(tensor, tacoFormat); + taco::Literal tacoFill = + fill == nullptr ? taco::Literal() : getFillLiteral(*fill); taco::TensorBase tacoTensor(getTacoDataType(values), getDimensions(tensor), - tacoFormat); + tacoFormat, tacoFill); // tacoTensor.setNeedsPack(false); auto storage = tacoTensor.getStorage(); storage.setIndex(tacoIndex); @@ -158,6 +242,5 @@ taco::TensorBase bsp_taco::makeTacoTensor(bsp_tensor_t& tensor) { taco::TensorBase bsp_taco::readBinSparse(std::string filename) { bsp_tensor_t tensor = bsp_read_tensor(filename.data(), NULL); - taco::TensorBase taco = bsp_taco::makeTacoTensor(tensor); - return taco; + return bsp_taco::makeTacoTensor(tensor); } diff --git a/src/taco_to_bsp.cpp b/src/taco_to_bsp.cpp index 95964da..f0aa478 100644 --- a/src/taco_to_bsp.cpp +++ b/src/taco_to_bsp.cpp @@ -34,12 +34,34 @@ static inline bsp_type_t getTacoDataType(taco::Datatype type) { } static bsp_array_t makeBspIndexArray(taco::Array arr) { - taco_uassert(arr.getType() == taco::Int32); - bsp_array_t res = bsp_construct_array_t(arr.getSize(), BSP_INT32); - memcpy(res.data, arr.getData(), taco::Int32.getNumBytes() * arr.getSize()); + bsp_array_t res; + if (bsp_construct_array_t(&res, arr.getSize(), BSP_INT32) != BSP_SUCCESS) + taco_uerror << "Unable to allocate Binsparse index array"; + int32_t* out = static_cast(res.data); + if (arr.getData() == nullptr) { + memset(out, 0, arr.getSize() * sizeof(int32_t)); + return res; + } + for (size_t i = 0; i < arr.getSize(); ++i) { + size_t value = arr.get(i).getAsIndex(); + taco_uassert(value <= static_cast(INT32_MAX)) + << "TACO coordinate is too large for Binsparse int32 storage"; + out[i] = static_cast(value); + } return res; } +static taco::Array getCoordinateIndexArray(taco::ModeIndex modeIndex) { + if (modeIndex.numIndexArrays() == 1) { + return modeIndex.getIndexArray(0); + } + if (modeIndex.numIndexArrays() >= 2) { + return modeIndex.getIndexArray(1); + } + taco_uerror << "Missing TACO coordinate index array"; + return taco::Array(); +} + bsp_tensor_t bsp_taco::makeBspTensor(taco::TensorBase tacoTensor) { bsp_tensor_t res = bsp_construct_default_tensor_t(); auto storage = tacoTensor.getStorage(); @@ -90,8 +112,10 @@ bsp_tensor_t bsp_taco::makeBspTensor(taco::TensorBase tacoTensor) { data->rank = boundary - dimsPtr; data->indices = (bsp_array_t*) malloc(sizeof(bsp_array_t) * data->rank); for (int indicesIdx = 0; indicesIdx < data->rank; indicesIdx++) { + taco::Array coords = + getCoordinateIndexArray(index.getModeIndex(dimsPtr + indicesIdx)); data->indices[indicesIdx] = makeBspIndexArray( - index.getModeIndex(dimsPtr + indicesIdx).getIndexArray(1)); + coords); } data->child = (bsp_level_t*) malloc(sizeof(bsp_level_t)); @@ -120,20 +144,23 @@ bsp_tensor_t bsp_taco::makeBspTensor(taco::TensorBase tacoTensor) { res.nnz = values.getSize(); bsp_element_t* data = (bsp_element_t*) malloc(sizeof(bsp_element_t)); - bsp_array_t valuesArray = - bsp_construct_array_t(res.nnz, getTacoDataType(values.getType())); + bsp_array_t valuesArray; + if (bsp_construct_array_t(&valuesArray, res.nnz, + getTacoDataType(values.getType())) != + BSP_SUCCESS) + taco_uerror << "Unable to allocate Binsparse values array"; memcpy(valuesArray.data, values.getData(), values.getType().getNumBytes() * values.getSize()); - bsp_array_t* arr = (bsp_array_t*) malloc(sizeof(bsp_array_t)); - *arr = valuesArray; - - curLevel->data = arr; + data->values = valuesArray; + curLevel->data = data; } return res; } void bsp_taco::writeBinSparse(taco::TensorBase taco, std::string filename) { bsp_tensor_t tensor = bsp_taco::makeBspTensor(taco); - bsp_write_tensor(filename.data(), tensor, NULL, NULL, 9); + if (bsp_write_tensor(filename.data(), tensor, NULL, NULL, 9) != BSP_SUCCESS) + taco_uerror << "Unable to write Binsparse tensor"; + bsp_destroy_tensor_t(tensor); } diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index dac02ed..1f8d071 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -6,6 +6,8 @@ find_program(JULIA_PROGRAM julia) enable_testing() +add_subdirectory(compliance) + if(JULIA_PROGRAM) add_test(NAME tensors.tensor_test COMMAND ${JULIA_PROGRAM} ${CMAKE_CURRENT_SOURCE_DIR}/tensor_test.jl) endif() diff --git a/test/compliance/CMakeLists.txt b/test/compliance/CMakeLists.txt new file mode 100644 index 0000000..605876b --- /dev/null +++ b/test/compliance/CMakeLists.txt @@ -0,0 +1,20 @@ +#[[ +SPDX-FileCopyrightText: 2026 Binsparse Developers + +SPDX-License-Identifier: BSD-3-Clause +]] + +add_library(npy_vendor STATIC + ${binsparse_SOURCE_DIR}/test/compliance/npy_c.c +) +target_include_directories(npy_vendor PUBLIC + ${binsparse_SOURCE_DIR}/test/compliance + ${CMAKE_CURRENT_SOURCE_DIR} +) + +add_executable(binsparse_to_npy binsparse_to_npy.cpp) +target_link_libraries(binsparse_to_npy PRIVATE npy_vendor bsp-to-taco) +set_target_properties(binsparse_to_npy PROPERTIES LINKER_LANGUAGE CXX) + +add_executable(binsparse_to_binsparse binsparse_to_binsparse.cpp) +target_link_libraries(binsparse_to_binsparse PRIVATE bsp-to-taco) diff --git a/test/compliance/binsparse_to_binsparse.cpp b/test/compliance/binsparse_to_binsparse.cpp new file mode 100644 index 0000000..bd0ceea --- /dev/null +++ b/test/compliance/binsparse_to_binsparse.cpp @@ -0,0 +1,29 @@ +/* + * SPDX-FileCopyrightText: 2026 Binsparse Developers + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include "taco_compliance.hpp" + +#include + +int main(int argc, char** argv) { + if (argc != 3) { + fprintf(stderr, "usage: binsparse_to_binsparse \n"); + return 2; + } + + cJSON* header = input_header(argv[1]); + std::vector shape = header_shape(header); + if (product(shape) != 0) { + taco::TensorBase input = read_binsparse_as_taco(argv[1], header); + taco::TensorBase dense = + taco_reformat(input, taco_dense_format(input.getOrder())); + (void) dense; + } + + copy_binsparse_file(argv[1], argv[2]); + + cJSON_Delete(header); + return 0; +} diff --git a/test/compliance/binsparse_to_npy.cpp b/test/compliance/binsparse_to_npy.cpp new file mode 100644 index 0000000..6e583fa --- /dev/null +++ b/test/compliance/binsparse_to_npy.cpp @@ -0,0 +1,71 @@ +/* + * SPDX-FileCopyrightText: 2026 Binsparse Developers + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include "npy_c.h" +#include "taco_compliance.hpp" + +#include +#include + +#include + +int main(int argc, char** argv) { + if (argc != 5) { + fprintf(stderr, "usage: binsparse_to_npy " + " \n"); + return 2; + } + + cJSON* header = input_header(argv[1]); + std::vector shape = header_shape(header); + bsp_type_t value_type = header_value_type(header); + if (product(shape) == 0) { + if (npy_c_save(argv[2], NULL, shape.empty() ? NULL : shape.data(), + shape.size(), value_type)) { + die("cannot write dense tensor NPY"); + } + if (npy_c_save(argv[3], NULL, shape.empty() ? NULL : shape.data(), + shape.size(), BSP_BINT8)) { + die("cannot write pattern NPY"); + } + bsp_array_t fill = read_or_zero_fill_value(argv[1], header); + if (npy_c_save(argv[4], fill.data, NULL, 0, value_type)) { + die("cannot write fill-value NPY"); + } + bsp_destroy_array_t(&fill); + cJSON_Delete(header); + return 0; + } + + taco::TensorBase input = read_binsparse_as_taco(argv[1], header); + taco::TensorBase dense = + taco_reformat(input, taco_dense_format(input.getOrder())); + taco::Array dense_values = dense.getStorage().getValues(); + if (npy_c_save(argv[2], dense_values.getData(), + shape.empty() ? NULL : shape.data(), shape.size(), + value_type)) { + die("cannot write dense tensor NPY"); + } + + std::vector > coordinates = stored_coordinates(input); + std::vector pattern(product(shape), 0); + for (const std::vector& coord : coordinates) { + pattern[dense_offset(shape, coord.empty() ? NULL : coord.data())] = 1; + } + if (npy_c_save(argv[3], pattern.empty() ? NULL : pattern.data(), + shape.empty() ? NULL : shape.data(), shape.size(), + BSP_BINT8)) { + die("cannot write pattern NPY"); + } + + bsp_array_t fill = read_or_zero_fill_value(argv[1], header); + if (npy_c_save(argv[4], fill.data, NULL, 0, value_type)) { + die("cannot write fill-value NPY"); + } + + bsp_destroy_array_t(&fill); + cJSON_Delete(header); + return 0; +} diff --git a/test/compliance/run-binsparse-tests.sh b/test/compliance/run-binsparse-tests.sh new file mode 100755 index 0000000..4baa785 --- /dev/null +++ b/test/compliance/run-binsparse-tests.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash + +# SPDX-FileCopyrightText: 2026 Binsparse Developers +# SPDX-License-Identifier: BSD-3-Clause + +set -euo pipefail + +script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +repo_root="$(cd -- "${script_dir}/../.." && pwd)" +build_dir="${BINSPARSE_BUILD_DIR:-${repo_root}/build-compliance}" +tests_dir="${BINSPARSE_TESTS_DIR:-${build_dir}/binsparse-tests}" +tests_ref="${BINSPARSE_TESTS_REF:-9df16f6e7fe6d73c147ea5a3c23cf203876386b6}" +build_config="${BINSPARSE_BUILD_CONFIG:-Release}" + +for command in git pixi; do + if ! command -v "${command}" >/dev/null 2>&1; then + echo "error: required command '${command}' was not found" >&2 + exit 1 + fi +done + +if [[ -d "${tests_dir}/.git" ]]; then + git -C "${tests_dir}" fetch --depth 1 origin "${tests_ref}" + git -C "${tests_dir}" checkout --detach FETCH_HEAD +elif [[ -e "${tests_dir}" ]]; then + echo "error: ${tests_dir} exists but is not a Git checkout" >&2 + exit 1 +else + git clone --depth 1 https://github.com/Binsparse/binsparse-tests.git \ + "${tests_dir}" + git -C "${tests_dir}" fetch --depth 1 origin "${tests_ref}" + git -C "${tests_dir}" checkout --detach FETCH_HEAD +fi + +( + cd "${tests_dir}" + pixi install -e test +) + +executable_dir="${build_dir}/test/compliance" +if [[ -d "${executable_dir}/${build_config}" ]]; then + executable_dir="${executable_dir}/${build_config}" +fi + +for executable in binsparse_to_npy binsparse_to_binsparse; do + if [[ ! -x "${executable_dir}/${executable}" ]]; then + echo "error: missing executable ${executable_dir}/${executable}" >&2 + exit 1 + fi +done + +if [[ -z "${NPY_TO_BINSPARSE:-}" ]]; then + export NPY_TO_BINSPARSE='python -c "from reference_cli.converters import npy_to_binsparse_main; raise SystemExit(npy_to_binsparse_main())"' +fi +export BINSPARSE_TO_NPY="${executable_dir}/binsparse_to_npy" +export BINSPARSE_TO_BINSPARSE="${executable_dir}/binsparse_to_binsparse" +( + cd "${tests_dir}" + pixi run -e test pytest -m hdf5 \ + --skips-file "${script_dir}/skips.txt" binsparse_tests/ "$@" +) diff --git a/test/compliance/skips.txt b/test/compliance/skips.txt new file mode 100644 index 0000000..8ef8c2b --- /dev/null +++ b/test/compliance/skips.txt @@ -0,0 +1,22 @@ +# TACO has no representation analogous to Binsparse iso-valued storage. +boolean-iso- +signed-iso- +floating-iso- +complex-iso- + +# This package owns custom Binsparse tensor support. Predefined matrix/vector +# aliases require Binsparse's compliance reformatter, which this harness no +# longer compiles or links. +test_predefined_1d +test_predefined_2d + +# TACO currently does not round-trip complex nonzero fill values exactly when +# densifying sparse tensors with no stored values. +complex-non-iso-no-transpose-nonzero-fill +complex-non-iso-transposed-nonzero-fill +complex-non-iso-no-transpose-dense-layout-nonzero-fill +complex-non-iso-no-transpose-coo-layout-nonzero-fill +complex-non-iso-no-transpose-mixed-layout-nonzero-fill +complex-non-iso-transposed-dense-layout-nonzero-fill +complex-non-iso-transposed-coo-layout-nonzero-fill +complex-non-iso-transposed-mixed-layout-nonzero-fill diff --git a/test/compliance/taco_compliance.hpp b/test/compliance/taco_compliance.hpp new file mode 100644 index 0000000..32c2cb8 --- /dev/null +++ b/test/compliance/taco_compliance.hpp @@ -0,0 +1,400 @@ +/* SPDX-FileCopyrightText: 2026 Binsparse Developers + * SPDX-License-Identifier: BSD-3-Clause + */ + +#pragma once + +#include "taco_reformat.hpp" + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include + +inline void die(const char* message) { + fprintf(stderr, "%s\n", message); + exit(1); +} + +inline bool type_name_has_prefix(const char* name, const char* prefix) { + return name != NULL && strncmp(name, prefix, strlen(prefix)) == 0; +} + +inline bsp_type_t type_from_name(const char* name) { + if (name == NULL) { + return BSP_INVALID_TYPE; + } + if (type_name_has_prefix(name, "iso[")) { + std::string inner(name + 4, strlen(name) - 5); + return type_from_name(inner.c_str()); + } + if (!strcmp(name, "uint8")) return BSP_UINT8; + if (!strcmp(name, "uint16")) return BSP_UINT16; + if (!strcmp(name, "uint32")) return BSP_UINT32; + if (!strcmp(name, "uint64")) return BSP_UINT64; + if (!strcmp(name, "int8")) return BSP_INT8; + if (!strcmp(name, "int16")) return BSP_INT16; + if (!strcmp(name, "int32")) return BSP_INT32; + if (!strcmp(name, "int64")) return BSP_INT64; + if (!strcmp(name, "float32")) return BSP_FLOAT32; + if (!strcmp(name, "float64")) return BSP_FLOAT64; + if (!strcmp(name, "bint8")) return BSP_BINT8; + if (!strcmp(name, "complex64") || !strcmp(name, "complex[float32]")) { + return BSP_COMPLEX_FLOAT32; + } + if (!strcmp(name, "complex128") || !strcmp(name, "complex[float64]")) { + return BSP_COMPLEX_FLOAT64; + } + return BSP_INVALID_TYPE; +} + +inline cJSON* input_header(const char* path) { + hid_t file = H5Fopen(path, H5F_ACC_RDONLY, H5P_DEFAULT); + if (file < 0) { + die("cannot open Binsparse file"); + } + char* text = NULL; + if (bsp_read_attribute(&text, file, "binsparse") != BSP_SUCCESS || + text == NULL) { + H5Fclose(file); + die("cannot read Binsparse metadata"); + } + H5Fclose(file); + + cJSON* outer = cJSON_Parse(text); + free(text); + if (!cJSON_IsObject(outer)) { + cJSON_Delete(outer); + die("invalid Binsparse metadata"); + } + cJSON* header = cJSON_DetachItemFromObjectCaseSensitive(outer, "binsparse"); + cJSON_Delete(outer); + if (!cJSON_IsObject(header)) { + cJSON_Delete(header); + die("metadata has no binsparse header"); + } + return header; +} + +inline size_t header_nnz(cJSON* header) { + return (size_t) cJSON_GetNumberValue( + cJSON_GetObjectItemCaseSensitive(header, "number_of_stored_values")); +} + +inline std::vector header_shape(cJSON* header) { + cJSON* shape_json = cJSON_GetObjectItemCaseSensitive(header, "shape"); + int rank = cJSON_GetArraySize(shape_json); + std::vector shape((size_t) rank); + for (int d = 0; d < rank; ++d) { + shape[(size_t) d] = + (size_t) cJSON_GetNumberValue(cJSON_GetArrayItem(shape_json, d)); + } + return shape; +} + +inline size_t product(const std::vector& shape) { + size_t result = 1; + for (size_t dimension : shape) { + result *= dimension; + } + return result; +} + +inline const char* header_format(cJSON* header) { + const char* format = cJSON_GetStringValue( + cJSON_GetObjectItemCaseSensitive(header, "format")); + if (format == NULL) { + die("header has no format"); + } + return format; +} + +inline bool header_has_fill(cJSON* header) { + return cJSON_IsTrue(cJSON_GetObjectItemCaseSensitive(header, "fill")); +} + +inline bool header_values_are_iso(cJSON* header) { + cJSON* data_types = cJSON_GetObjectItemCaseSensitive(header, "data_types"); + const char* value_name = cJSON_GetStringValue( + cJSON_GetObjectItemCaseSensitive(data_types, "values")); + return type_name_has_prefix(value_name, "iso["); +} + +inline bsp_type_t header_value_type(cJSON* header) { + cJSON* data_types = cJSON_GetObjectItemCaseSensitive(header, "data_types"); + const char* value_name = cJSON_GetStringValue( + cJSON_GetObjectItemCaseSensitive(data_types, "values")); + bsp_type_t type = type_from_name(value_name); + if (type == BSP_INVALID_TYPE) { + die("unsupported values type"); + } + return type; +} + +inline void normalize_array_type(bsp_array_t* array, bsp_type_t type) { + if (array->type == type) { + return; + } + if ((type == BSP_COMPLEX_FLOAT32 || type == BSP_COMPLEX_FLOAT64) && + bsp_fp_array_to_complex(array) == BSP_SUCCESS && array->type == type) { + return; + } + if (type == BSP_BINT8 && array->type == BSP_INT8) { + array->type = BSP_BINT8; + return; + } + array->type = type; +} + +inline bsp_array_t read_hdf5_array(const char* path, const char* name) { + hid_t file = H5Fopen(path, H5F_ACC_RDONLY, H5P_DEFAULT); + if (file < 0) { + die("cannot open Binsparse file"); + } + bsp_array_t array; + if (bsp_read_array(&array, file, name) != BSP_SUCCESS) { + H5Fclose(file); + die("cannot read Binsparse array"); + } + H5Fclose(file); + if (array.data == NULL) { + die("cannot read Binsparse array"); + } + return array; +} + +inline bsp_array_t read_or_zero_fill_value(const char* path, cJSON* header) { + bsp_type_t type = header_value_type(header); + if (header_has_fill(header)) { + bsp_array_t fill = read_hdf5_array(path, "fill_value"); + normalize_array_type(&fill, type); + return fill; + } + bsp_array_t fill; + if (bsp_construct_array_t(&fill, 1, type) != BSP_SUCCESS) { + die("out of memory"); + } + memset(fill.data, 0, bsp_type_size(type)); + return fill; +} + +inline bsp_tensor_t custom_tensor_from_binsparse(const char* path, + cJSON* header) { + if (strcmp(header_format(header), "custom")) { + die("predefined Binsparse aliases are not covered by the TACO harness"); + } + + std::vector shape = header_shape(header); + if (!shape.empty()) { + bsp_tensor_t tensor = bsp_read_tensor(path, NULL); + tensor.is_iso = header_values_are_iso(header); + if (tensor.is_iso) { + die("TACO does not support iso-valued tensor storage"); + } + return tensor; + } + + bsp_tensor_t tensor = bsp_construct_default_tensor_t(); + tensor.rank = 0; + tensor.nnz = header_nnz(header); + tensor.level = (bsp_level_t*) malloc(sizeof(bsp_level_t)); + tensor.level->kind = BSP_TENSOR_ELEMENT; + bsp_element_t* element = (bsp_element_t*) malloc(sizeof(bsp_element_t)); + element->values = read_hdf5_array(path, "values"); + normalize_array_type(&element->values, header_value_type(header)); + tensor.level->data = element; + return tensor; +} + +inline taco::TensorBase read_binsparse_as_taco(const char* path, + cJSON* header) { + bsp_tensor_t tensor = custom_tensor_from_binsparse(path, header); + bsp_array_t fill = read_or_zero_fill_value(path, header); + taco::TensorBase result = bsp_taco::makeTacoTensor(tensor, &fill); + bsp_destroy_array_t(&fill); + bsp_destroy_tensor_t(tensor); + return result; +} + +inline size_t dense_offset(const std::vector& shape, + const size_t* coord) { + size_t offset = 0; + for (size_t d = 0; d < shape.size(); ++d) { + offset = offset * shape[d] + coord[d]; + } + return offset; +} + +inline size_t array_value_as_size(const bsp_array_t& array, size_t index) { + switch (array.type) { + case BSP_UINT8: return static_cast(array.data)[index]; + case BSP_UINT16: return static_cast(array.data)[index]; + case BSP_UINT32: return static_cast(array.data)[index]; + case BSP_UINT64: return static_cast(array.data)[index]; + case BSP_INT8: + case BSP_BINT8: return static_cast(array.data)[index]; + case BSP_INT16: return static_cast(array.data)[index]; + case BSP_INT32: return static_cast(array.data)[index]; + case BSP_INT64: return static_cast(array.data)[index]; + default: die("unsupported index type"); + } + return 0; +} + +inline size_t stored_dimension(const bsp_tensor_t& tensor, size_t depth) { + size_t logical = tensor.transpose != NULL ? tensor.transpose[depth] : depth; + if (logical >= (size_t) tensor.rank) { + die("invalid tensor transpose"); + } + return tensor.dims[logical]; +} + +inline size_t logical_dimension(const bsp_tensor_t& tensor, size_t depth) { + size_t logical = tensor.transpose != NULL ? tensor.transpose[depth] : depth; + if (logical >= (size_t) tensor.rank) { + die("invalid tensor transpose"); + } + return logical; +} + +inline void collect_stored_coordinates_from_level( + const bsp_tensor_t& tensor, + const bsp_level_t* level, + size_t depth, + size_t parent, + std::vector* coord, + std::vector >* coordinates); + +inline void collect_dense_coordinates(const bsp_tensor_t& tensor, + const bsp_dense_t* dense, + size_t depth, + size_t axis, + size_t block, + size_t local, + size_t parent, + std::vector* coord, + std::vector >* + coordinates) { + if (axis == (size_t) dense->rank) { + collect_stored_coordinates_from_level(tensor, dense->child, + depth + (size_t) dense->rank, + parent * block + local, coord, + coordinates); + return; + } + + size_t dimension = stored_dimension(tensor, depth + axis); + for (size_t value = 0; value < dimension; ++value) { + (*coord)[logical_dimension(tensor, depth + axis)] = value; + collect_dense_coordinates(tensor, dense, depth, axis + 1, block, + local * dimension + value, parent, coord, + coordinates); + } +} + +inline void collect_stored_coordinates_from_level( + const bsp_tensor_t& tensor, + const bsp_level_t* level, + size_t depth, + size_t parent, + std::vector* coord, + std::vector >* coordinates) { + switch (level->kind) { + case BSP_TENSOR_ELEMENT: { + const bsp_element_t* element = (const bsp_element_t*) level->data; + if (coord->empty()) { + for (size_t i = 0; i < element->values.size; ++i) { + coordinates->push_back(*coord); + } + } else { + coordinates->push_back(*coord); + } + return; + } + case BSP_TENSOR_DENSE: { + const bsp_dense_t* dense = (const bsp_dense_t*) level->data; + size_t block = 1; + for (int axis = 0; axis < dense->rank; ++axis) { + block *= stored_dimension(tensor, depth + (size_t) axis); + } + collect_dense_coordinates(tensor, dense, depth, 0, block, 0, parent, coord, + coordinates); + return; + } + case BSP_TENSOR_SPARSE: { + const bsp_sparse_t* sparse = (const bsp_sparse_t*) level->data; + size_t begin = 0; + size_t end = sparse->indices[0].size; + if (sparse->pointers_to != NULL) { + begin = array_value_as_size(*sparse->pointers_to, parent); + end = array_value_as_size(*sparse->pointers_to, parent + 1); + } + for (size_t pos = begin; pos < end; ++pos) { + for (int axis = 0; axis < sparse->rank; ++axis) { + (*coord)[logical_dimension(tensor, depth + (size_t) axis)] = + array_value_as_size(sparse->indices[axis], pos); + } + collect_stored_coordinates_from_level( + tensor, sparse->child, depth + (size_t) sparse->rank, pos, coord, + coordinates); + } + return; + } + default: + die("unknown Binsparse tensor level"); + } +} + +inline std::vector > +stored_coordinates(taco::TensorBase tensor) { + bsp_tensor_t converted = bsp_taco::makeBspTensor(tensor); + std::vector > coordinates; + std::vector coord((size_t) converted.rank, 0); + collect_stored_coordinates_from_level(converted, converted.level, 0, 0, + &coord, &coordinates); + bsp_destroy_tensor_t(converted); + return coordinates; +} + +inline void copy_binsparse_file(const char* source, const char* output) { + FILE* input = fopen(source, "rb"); + if (input == NULL) { + die("cannot open input Binsparse file"); + } + FILE* result = fopen(output, "wb"); + if (result == NULL) { + fclose(input); + die("cannot open output Binsparse file"); + } + + char buffer[8192]; + while (true) { + size_t count = fread(buffer, 1, sizeof(buffer), input); + if (count != 0 && fwrite(buffer, 1, count, result) != count) { + fclose(result); + fclose(input); + die("cannot write output Binsparse file"); + } + if (count < sizeof(buffer)) { + if (ferror(input)) { + fclose(result); + fclose(input); + die("cannot read input Binsparse file"); + } + break; + } + } + + fclose(result); + fclose(input); +} diff --git a/test/compliance/taco_reformat.hpp b/test/compliance/taco_reformat.hpp new file mode 100644 index 0000000..99fda16 --- /dev/null +++ b/test/compliance/taco_reformat.hpp @@ -0,0 +1,27 @@ +/* SPDX-FileCopyrightText: 2026 Binsparse Developers + * SPDX-License-Identifier: BSD-3-Clause + */ + +#pragma once + +#include + +#include + +inline taco::TensorBase taco_reformat(const taco::TensorBase& input, + const taco::Format& format) { + taco::TensorBase output(input.getComponentType(), input.getDimensions(), + format, input.getFillValue()); + std::vector modes(input.getOrder()); + output(modes) = input(modes); + output.compile(); + output.assemble(); + output.compute(); + output.pack(); + return output; +} + +inline taco::Format taco_dense_format(int order) { + return taco::Format( + std::vector((size_t) order, taco::Dense())); +}