From d3de94791e93160b09bcba83c46e874a773bf8e8 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Wed, 5 Aug 2026 16:11:02 +0800 Subject: [PATCH 1/4] Add C++ and Python TsFile properties support --- cpp/README-zh.md | 19 ++ cpp/README.md | 20 +++ cpp/src/common/tsfile_common.cc | 68 ++++++- cpp/src/common/tsfile_common.h | 30 +++- cpp/src/cwrapper/tsfile_cwrapper.cc | 144 +++++++++++++++ cpp/src/cwrapper/tsfile_cwrapper.h | 42 +++++ cpp/src/file/tsfile_io_writer.cc | 48 ++++- cpp/src/file/tsfile_io_writer.h | 5 + cpp/src/reader/tsfile_reader.cc | 9 + cpp/src/reader/tsfile_reader.h | 3 + cpp/src/writer/tsfile_table_writer.cc | 17 ++ cpp/src/writer/tsfile_table_writer.h | 6 + cpp/src/writer/tsfile_writer.cc | 17 ++ cpp/src/writer/tsfile_writer.h | 6 + cpp/test/common/tsfile_common_test.cc | 13 +- cpp/test/cwrapper/cwrapper_properties_test.cc | 166 ++++++++++++++++++ .../table_view/tsfile_writer_table_test.cc | 24 +++ cpp/test/writer/tsfile_properties_test.cc | 112 ++++++++++++ python/README-zh.md | 18 +- python/README.md | 16 ++ python/tests/test_tsfile_properties.py | 79 +++++++++ python/tsfile/tsfile_cpp.pxd | 16 ++ python/tsfile/tsfile_reader.pyx | 36 +++- python/tsfile/tsfile_table_writer.py | 6 + python/tsfile/tsfile_writer.pyx | 37 +++- 25 files changed, 929 insertions(+), 28 deletions(-) create mode 100644 cpp/test/cwrapper/cwrapper_properties_test.cc create mode 100644 cpp/test/writer/tsfile_properties_test.cc create mode 100644 python/tests/test_tsfile_properties.py diff --git a/cpp/README-zh.md b/cpp/README-zh.md index a8af952b1..b0d76f506 100644 --- a/cpp/README-zh.md +++ b/cpp/README-zh.md @@ -188,3 +188,22 @@ bash build.sh ``` 即可在 `./examples/build` 目录下生成可执行文件。 + +### 文件级 Properties + +`TsFileWriter` 和 `TsFileTableWriter` 可以在 writer 打开期间新增或覆盖二进制 +property。传入的数据会立即复制，调用 `flush()` 后仍可继续修改；文件关闭后不能修改。 + +```cpp +std::vector value = {0x01, 0x00, 0xFF}; +writer.add_tsfile_property("binary-property", value); + +// nullptr 且长度为 0 表示 null；空 vector 表示非 null 的零长度值。 +writer.add_tsfile_property("null-property", nullptr, 0); +writer.add_tsfile_property("empty-property", std::vector()); + +storage::TsFileProperties properties = reader.get_tsfile_properties(); +``` + +Property value 本身不保存数据类型。整数、浮点数或结构体应由应用使用明确、可跨语言的 +字节编码进行转换。 diff --git a/cpp/README.md b/cpp/README.md index 918eff68f..5f62afe24 100644 --- a/cpp/README.md +++ b/cpp/README.md @@ -203,3 +203,23 @@ By default, parallel write is enabled when the machine has more than one CPU cor ## Use TsFile You can find examples on how to read and write data in `demo_read.cpp` and `demo_write.cpp` located under `./examples/cpp_examples`. There are also examples under `./examples/c_examples` on how to use a C-style API to read and write data in a C environment. The examples will be built automatically when you run the main build command. + +### File-level properties + +`TsFileWriter` and `TsFileTableWriter` can add or replace binary properties +while the writer is open. Values are copied immediately and may still be +changed after `flush()`; a closed file cannot be modified. + +```cpp +std::vector value = {0x01, 0x00, 0xFF}; +writer.add_tsfile_property("binary-property", value); + +// nullptr with length 0 is null; an empty vector is a non-null empty value. +writer.add_tsfile_property("null-property", nullptr, 0); +writer.add_tsfile_property("empty-property", std::vector()); + +storage::TsFileProperties properties = reader.get_tsfile_properties(); +``` + +Property values do not store a data type. Applications should define their own +portable byte encoding for integers, floating-point values, or structures. diff --git a/cpp/src/common/tsfile_common.cc b/cpp/src/common/tsfile_common.cc index a3fcc0a70..f0d97dc00 100644 --- a/cpp/src/common/tsfile_common.cc +++ b/cpp/src/common/tsfile_common.cc @@ -206,8 +206,16 @@ int TsFileMeta::serialize_to(common::ByteStream& out) { common::SerializationUtil::write_var_int(tsfile_properties_.size(), out); for (const auto& tsfile_property : tsfile_properties_) { common::SerializationUtil::write_var_str(tsfile_property.first, out); - common::SerializationUtil::write_var_char_ptr(tsfile_property.second, - out); + const TsFilePropertyValue& value = tsfile_property.second; + if (value.is_null) { + common::SerializationUtil::write_var_int(NO_STR_TO_READ, out); + } else { + common::SerializationUtil::write_var_int( + static_cast(value.value.size()), out); + if (!value.value.empty()) { + out.write_buf(value.value.data(), value.value.size()); + } + } } return out.total_size() - start_idx; @@ -254,12 +262,56 @@ int TsFileMeta::deserialize_from(common::ByteStream& in) { bloom_filter_->deserialize_from(in); int32_t tsfile_properties_size = 0; - common::SerializationUtil::read_var_int(tsfile_properties_size, in); + if (RET_FAIL(common::SerializationUtil::read_var_int(tsfile_properties_size, + in))) { + return ret; + } + if (tsfile_properties_size < 0) { + return common::E_TSFILE_CORRUPTED; + } for (int i = 0; i < tsfile_properties_size; i++) { - std::string key, *value; - common::SerializationUtil::read_var_str(key, in); - common::SerializationUtil::read_var_char_ptr(value, in); - tsfile_properties_.emplace(key, value); + std::string key; + int32_t key_len = 0; + int32_t value_len = 0; + if (RET_FAIL(common::SerializationUtil::read_var_int(key_len, in))) { + return ret; + } else if (key_len < 0) { + return common::E_TSFILE_CORRUPTED; + } + key.resize(static_cast(key_len)); + if (key_len > 0) { + uint32_t read_len = 0; + if (RET_FAIL(in.read_buf(reinterpret_cast(&key[0]), + static_cast(key_len), + read_len))) { + return ret; + } else if (read_len != static_cast(key_len)) { + return common::E_BUF_NOT_ENOUGH; + } + } + if (RET_FAIL(common::SerializationUtil::read_var_int(value_len, in))) { + return ret; + } + + TsFilePropertyValue value; + if (value_len == NO_STR_TO_READ) { + value.is_null = true; + } else if (value_len < 0) { + return common::E_TSFILE_CORRUPTED; + } else { + value.is_null = false; + value.value.resize(static_cast(value_len)); + if (value_len > 0) { + uint32_t read_len = 0; + if (RET_FAIL( + in.read_buf(value.value.data(), value_len, read_len))) { + return ret; + } else if (read_len != static_cast(value_len)) { + return common::E_BUF_NOT_ENOUGH; + } + } + } + tsfile_properties_.emplace(key, std::move(value)); } return ret; } @@ -375,4 +427,4 @@ int MetaIndexNode::binary_search_children(const String &name, } #endif -} // end namespace storage \ No newline at end of file +} // end namespace storage diff --git a/cpp/src/common/tsfile_common.h b/cpp/src/common/tsfile_common.h index fd3690200..811a11f22 100644 --- a/cpp/src/common/tsfile_common.h +++ b/cpp/src/common/tsfile_common.h @@ -1126,13 +1126,35 @@ struct MetaIndexNode { class TableSchema; +struct TsFilePropertyValue { + /** A default-constructed property represents a null value. */ + TsFilePropertyValue() : is_null(true), value() {} + + /** A vector, including an empty vector, represents a non-null value. */ + explicit TsFilePropertyValue(const std::vector& value) + : is_null(false), value(value) {} + + /** nullptr represents null; a non-null pointer with length 0 is empty. */ + TsFilePropertyValue(const uint8_t* data, uint32_t value_len) + : is_null(data == nullptr), value() { + if (data != nullptr && value_len > 0) { + value.assign(data, data + value_len); + } + } + + bool is_null; + std::vector value; +}; + +using TsFileProperties = std::unordered_map; + struct TsFileMeta { typedef std::map, std::shared_ptr, IDeviceIDComparator> DeviceNodeMap; std::map> table_metadata_index_node_map_; - std::unordered_map tsfile_properties_; + TsFileProperties tsfile_properties_; typedef std::unordered_map> TableSchemasMap; TableSchemasMap table_schemas_; @@ -1170,12 +1192,6 @@ struct TsFileMeta { if (bloom_filter_ != nullptr) { bloom_filter_->destroy(); } - for (auto properties : tsfile_properties_) { - if (properties.second != nullptr) { - delete properties.second; - properties.second = nullptr; - } - } tsfile_properties_.clear(); table_metadata_index_node_map_.clear(); table_schemas_.clear(); diff --git a/cpp/src/cwrapper/tsfile_cwrapper.cc b/cpp/src/cwrapper/tsfile_cwrapper.cc index ffe2f59b6..a969cfc9a 100644 --- a/cpp/src/cwrapper/tsfile_cwrapper.cc +++ b/cpp/src/cwrapper/tsfile_cwrapper.cc @@ -31,6 +31,8 @@ #endif #include +#include +#include #include #include @@ -258,6 +260,25 @@ ERRNO tsfile_writer_close(TsFileWriter writer) { return ret; } +ERRNO tsfile_writer_add_tsfile_property(TsFileWriter writer, const char* key, + uint32_t key_len, const uint8_t* value, + uint32_t value_len) { + if (writer == nullptr || key == nullptr || + (value == nullptr && value_len > 0)) { + return common::E_INVALID_ARG; + } + if (key_len > static_cast(std::numeric_limits::max())) { + return common::E_OUT_OF_RANGE; + } + try { + auto* w = static_cast(writer); + return w->add_tsfile_property(std::string(key, key_len), value, + value_len); + } catch (const std::bad_alloc&) { + return common::E_OOM; + } +} + ERRNO tsfile_reader_close(TsFileReader reader) { auto* ts_reader = static_cast(reader); delete ts_reader; @@ -1432,6 +1453,110 @@ void tsfile_free_device_timeseries_metadata_map( map->device_count = 0; } +void tsfile_free_tsfile_properties(TsFileProperty* properties, + uint32_t length) { + if (properties == nullptr) { + return; + } + for (uint32_t i = 0; i < length; i++) { + free(properties[i].key); + properties[i].key = nullptr; + free(properties[i].value); + properties[i].value = nullptr; + properties[i].key_len = 0; + properties[i].value_len = 0; + properties[i].is_null = false; + } + free(properties); +} + +ERRNO tsfile_reader_get_tsfile_properties(TsFileReader reader, + TsFileProperty** out_properties, + uint32_t* out_length) { + if (out_properties == nullptr || out_length == nullptr) { + return common::E_INVALID_ARG; + } + *out_properties = nullptr; + *out_length = 0; + if (reader == nullptr) { + return common::E_INVALID_ARG; + } + + try { + auto* r = static_cast(reader); + storage::TsFileProperties cpp_properties = r->get_tsfile_properties(); + if (cpp_properties.size() > + static_cast(std::numeric_limits::max()) || + cpp_properties.size() > + std::numeric_limits::max() / sizeof(TsFileProperty)) { + return common::E_OUT_OF_RANGE; + } + if (cpp_properties.empty()) { + return common::E_OK; + } + + auto* properties = static_cast( + malloc(sizeof(TsFileProperty) * cpp_properties.size())); + if (properties == nullptr) { + return common::E_OOM; + } + memset(properties, 0, sizeof(TsFileProperty) * cpp_properties.size()); + + uint32_t property_index = 0; + for (const auto& cpp_property : cpp_properties) { + TsFileProperty& property = properties[property_index]; + if (cpp_property.first.size() > + static_cast(std::numeric_limits::max())) { + tsfile_free_tsfile_properties(properties, property_index); + return common::E_OUT_OF_RANGE; + } + property.key_len = static_cast(cpp_property.first.size()); + property.key = static_cast( + malloc(static_cast(property.key_len) + 1U)); + if (property.key == nullptr) { + tsfile_free_tsfile_properties(properties, property_index + 1); + return common::E_OOM; + } + if (property.key_len > 0) { + memcpy(property.key, cpp_property.first.data(), + property.key_len); + } + property.key[property.key_len] = '\0'; + + const storage::TsFilePropertyValue& cpp_value = cpp_property.second; + property.is_null = cpp_value.is_null; + if (!cpp_value.is_null) { + if (cpp_value.value.size() > + static_cast(std::numeric_limits::max())) { + tsfile_free_tsfile_properties(properties, + property_index + 1); + return common::E_OUT_OF_RANGE; + } + property.value_len = + static_cast(cpp_value.value.size()); + if (property.value_len > 0) { + property.value = + static_cast(malloc(property.value_len)); + if (property.value == nullptr) { + tsfile_free_tsfile_properties(properties, + property_index + 1); + return common::E_OOM; + } + memcpy(property.value, cpp_value.value.data(), + property.value_len); + } + } + property_index++; + } + + *out_properties = properties; + *out_length = static_cast(cpp_properties.size()); + return common::E_OK; + } catch (const std::bad_alloc&) { + return common::E_OOM; + } +} + // delete pointer void _free_tsfile_ts_record(TsRecord* record) { if (*record != nullptr) { @@ -1640,6 +1765,25 @@ ERRNO _tsfile_writer_flush(TsFileWriter writer) { return w->flush(); } +ERRNO _tsfile_writer_add_tsfile_property(TsFileWriter writer, const char* key, + uint32_t key_len, const uint8_t* value, + uint32_t value_len) { + if (writer == nullptr || key == nullptr || + (value == nullptr && value_len > 0)) { + return common::E_INVALID_ARG; + } + if (key_len > static_cast(std::numeric_limits::max())) { + return common::E_OUT_OF_RANGE; + } + try { + auto* w = static_cast(writer); + return w->add_tsfile_property(std::string(key, key_len), value, + value_len); + } catch (const std::bad_alloc&) { + return common::E_OOM; + } +} + ResultSet _tsfile_reader_query_device(TsFileReader reader, const char* device_name, char** sensor_name, uint32_t sensor_num, diff --git a/cpp/src/cwrapper/tsfile_cwrapper.h b/cpp/src/cwrapper/tsfile_cwrapper.h index 768aec962..16d392bae 100644 --- a/cpp/src/cwrapper/tsfile_cwrapper.h +++ b/cpp/src/cwrapper/tsfile_cwrapper.h @@ -230,6 +230,21 @@ typedef struct DeviceTimeseriesMetadataMap { uint32_t device_count; } DeviceTimeseriesMetadataMap; +/** + * @brief One file-level property with length-aware binary storage. + * + * @p key is allocated with one trailing NUL for convenience, while @p key_len + * is authoritative and preserves embedded NUL bytes. @p is_null distinguishes + * a null value from a non-null zero-length value. + */ +typedef struct TsFileProperty { + char* key; + uint32_t key_len; + uint8_t* value; + uint32_t value_len; + bool is_null; +} TsFileProperty; + /** Frees path, table_name, and segments inside @p d; zeros @p d. */ void tsfile_device_id_free_contents(DeviceID* d); @@ -435,6 +450,17 @@ TsFileReader tsfile_reader_new(const char* pathname, ERRNO* err_code); */ ERRNO tsfile_writer_close(TsFileWriter writer); +/** + * @brief Adds or replaces a file-level property while the table writer is open. + * + * The key and value are copied immediately. A NULL value with value_len == 0 + * represents a null property; a non-NULL value with value_len == 0 represents + * an empty byte array. + */ +ERRNO tsfile_writer_add_tsfile_property(TsFileWriter writer, const char* key, + uint32_t key_len, const uint8_t* value, + uint32_t value_len); + /** * @brief Releases resources associated with a TsFileReader. * @@ -477,6 +503,17 @@ ERRNO tsfile_reader_get_timeseries_metadata_for_devices( void tsfile_free_device_timeseries_metadata_map( DeviceTimeseriesMetadataMap* map); +/** + * @brief Returns a heap-allocated array containing all file-level properties. + * + * Caller must release the result with tsfile_free_tsfile_properties(). + */ +ERRNO tsfile_reader_get_tsfile_properties(TsFileReader reader, + TsFileProperty** out_properties, + uint32_t* out_length); + +void tsfile_free_tsfile_properties(TsFileProperty* properties, uint32_t length); + /*--------------------------Tablet API------------------------ */ /** @@ -1055,6 +1092,11 @@ ERRNO _tsfile_writer_close(TsFileWriter writer); // Flush Chunk into tsfile from current tsFileWriter ERRNO _tsfile_writer_flush(TsFileWriter writer); +// Add or replace a file-level property on the generic writer used by Python. +ERRNO _tsfile_writer_add_tsfile_property(TsFileWriter writer, const char* key, + uint32_t key_len, const uint8_t* value, + uint32_t value_len); + // Queries time-series data for a specific device within a given time range. ResultSet _tsfile_reader_query_device(TsFileReader reader, const char* device_name, diff --git a/cpp/src/file/tsfile_io_writer.cc b/cpp/src/file/tsfile_io_writer.cc index 8c207ca82..37221dd95 100644 --- a/cpp/src/file/tsfile_io_writer.cc +++ b/cpp/src/file/tsfile_io_writer.cc @@ -23,6 +23,7 @@ #include #include +#include #include #include "common/device_id.h" @@ -93,6 +94,7 @@ void TsFileIOWriter::destroy() { use_prev_alloc_cgm_ = false; is_aligned_ = false; file_base_offset_ = 0; + tsfile_properties_.clear(); destroyed_ = true; meta_allocator_.destroy(); @@ -103,6 +105,38 @@ void TsFileIOWriter::destroy() { } } +int TsFileIOWriter::add_tsfile_property(const std::string& key, + const uint8_t* value, + uint32_t value_len) { + if (file_ == nullptr || file_->get_fd() < 0) { + return common::E_FILE_WRITE_ERR; + } + if (value_len > 0 && value == nullptr) { + return common::E_INVALID_ARG; + } + if (key.size() > static_cast(std::numeric_limits::max()) || + value_len > + static_cast(std::numeric_limits::max())) { + return common::E_OUT_OF_RANGE; + } + tsfile_properties_[key] = TsFilePropertyValue(value, value_len); + return common::E_OK; +} + +int TsFileIOWriter::add_tsfile_property(const std::string& key, + const std::vector& value) { + if (key.size() > static_cast(std::numeric_limits::max()) || + value.size() > + static_cast(std::numeric_limits::max())) { + return common::E_OUT_OF_RANGE; + } + if (file_ == nullptr || file_->get_fd() < 0) { + return common::E_FILE_WRITE_ERR; + } + tsfile_properties_[key] = TsFilePropertyValue(value); + return common::E_OK; +} + int TsFileIOWriter::start_file() { int ret = E_OK; if (RET_FAIL(write_buf(MAGIC_STRING_TSFILE, MAGIC_STRING_TSFILE_LEN))) { @@ -472,12 +506,14 @@ int TsFileIOWriter::write_file_index() { } tsfile_meta.table_metadata_index_node_map_ = table_nodes_map; tsfile_meta.table_schemas_ = schema_->table_schema_map_; - tsfile_meta.tsfile_properties_.insert( - std::make_pair("encryptLevel", new std::string(encrypt_level_))); - tsfile_meta.tsfile_properties_.insert( - std::make_pair("encryptType", new std::string(encrypt_type_))); - tsfile_meta.tsfile_properties_.insert( - std::make_pair("encryptKey", nullptr)); + tsfile_meta.tsfile_properties_ = tsfile_properties_; + tsfile_meta.tsfile_properties_["encryptLevel"] = TsFilePropertyValue( + reinterpret_cast(encrypt_level_.data()), + static_cast(encrypt_level_.size())); + tsfile_meta.tsfile_properties_["encryptType"] = TsFilePropertyValue( + reinterpret_cast(encrypt_type_.data()), + static_cast(encrypt_type_.size())); + tsfile_meta.tsfile_properties_["encryptKey"] = TsFilePropertyValue(); #if DEBUG_SE auto tsfile_meta_offset = write_stream_.total_size(); #endif diff --git a/cpp/src/file/tsfile_io_writer.h b/cpp/src/file/tsfile_io_writer.h index f041a1c57..bbb1e4988 100644 --- a/cpp/src/file/tsfile_io_writer.h +++ b/cpp/src/file/tsfile_io_writer.h @@ -89,6 +89,10 @@ class TsFileIOWriter { void destroy(); void set_generate_table_schema(bool generate_table_schema); + int add_tsfile_property(const std::string& key, const uint8_t* value, + uint32_t value_len); + int add_tsfile_property(const std::string& key, + const std::vector& value); int start_file(); int start_flush_chunk_group(std::shared_ptr device_id, bool is_aligned = false); @@ -242,6 +246,7 @@ class TsFileIOWriter { std::string encrypt_level_; std::string encrypt_type_; std::string encrypt_key_; + TsFileProperties tsfile_properties_; bool is_aligned_; /** Recovery only: absolute file offset at which write_stream_ logically * begins. Normal (non-recovery) path keeps this at 0. */ diff --git a/cpp/src/reader/tsfile_reader.cc b/cpp/src/reader/tsfile_reader.cc index 33d0d8967..fb5f8fd92 100644 --- a/cpp/src/reader/tsfile_reader.cc +++ b/cpp/src/reader/tsfile_reader.cc @@ -527,6 +527,15 @@ DeviceTimeseriesMetadataMap TsFileReader::get_timeseries_metadata() { return result; } +TsFileProperties TsFileReader::get_tsfile_properties() { + if (tsfile_executor_ == nullptr) { + return TsFileProperties(); + } + TsFileMeta* file_metadata = tsfile_executor_->get_tsfile_meta(); + return file_metadata == nullptr ? TsFileProperties() + : file_metadata->tsfile_properties_; +} + ResultSet* TsFileReader::read_timeseries( const std::shared_ptr& device_id, const std::vector& measurement_name) { diff --git a/cpp/src/reader/tsfile_reader.h b/cpp/src/reader/tsfile_reader.h index e2f9f3496..3ba490045 100644 --- a/cpp/src/reader/tsfile_reader.h +++ b/cpp/src/reader/tsfile_reader.h @@ -216,6 +216,9 @@ class TsFileReader { */ DeviceTimeseriesMetadataMap get_timeseries_metadata(); + /** Return a copy of all file-level properties, preserving null values. */ + TsFileProperties get_tsfile_properties(); + /** * @brief get the table schema by the table name * diff --git a/cpp/src/writer/tsfile_table_writer.cc b/cpp/src/writer/tsfile_table_writer.cc index b1b7911bd..5432aff5c 100644 --- a/cpp/src/writer/tsfile_table_writer.cc +++ b/cpp/src/writer/tsfile_table_writer.cc @@ -92,6 +92,23 @@ int storage::TsFileTableWriter::flush() { return tsfile_writer_->flush(); } +int storage::TsFileTableWriter::add_tsfile_property(const std::string& key, + const uint8_t* value, + uint32_t value_len) { + if (closed_ || !tsfile_writer_) { + return common::E_FILE_WRITE_ERR; + } + return tsfile_writer_->add_tsfile_property(key, value, value_len); +} + +int storage::TsFileTableWriter::add_tsfile_property( + const std::string& key, const std::vector& value) { + if (closed_ || !tsfile_writer_) { + return common::E_FILE_WRITE_ERR; + } + return tsfile_writer_->add_tsfile_property(key, value); +} + int storage::TsFileTableWriter::close() { if (closed_) { return common::E_OK; diff --git a/cpp/src/writer/tsfile_table_writer.h b/cpp/src/writer/tsfile_table_writer.h index a2d2a5fd9..d7c79254f 100644 --- a/cpp/src/writer/tsfile_table_writer.h +++ b/cpp/src/writer/tsfile_table_writer.h @@ -106,6 +106,12 @@ class TsFileTableWriter { * @return Returns 0 on success, or a non-zero error code on failure. */ int flush(); + + /** Add or replace a binary file-level property while the writer is open. */ + int add_tsfile_property(const std::string& key, const uint8_t* value, + uint32_t value_len); + int add_tsfile_property(const std::string& key, + const std::vector& value); /** * Closes the writer and releases any resources held by it. * After calling this method, no further operations should be performed on diff --git a/cpp/src/writer/tsfile_writer.cc b/cpp/src/writer/tsfile_writer.cc index 0b4c8668c..aa0e555f8 100644 --- a/cpp/src/writer/tsfile_writer.cc +++ b/cpp/src/writer/tsfile_writer.cc @@ -1960,4 +1960,21 @@ int TsFileWriter::close() { return io_writer_->end_file(); } +int TsFileWriter::add_tsfile_property(const std::string& key, + const uint8_t* value, + uint32_t value_len) { + if (io_writer_ == nullptr) { + return E_FILE_WRITE_ERR; + } + return io_writer_->add_tsfile_property(key, value, value_len); +} + +int TsFileWriter::add_tsfile_property(const std::string& key, + const std::vector& value) { + if (io_writer_ == nullptr) { + return E_FILE_WRITE_ERR; + } + return io_writer_->add_tsfile_property(key, value); +} + } // end namespace storage diff --git a/cpp/src/writer/tsfile_writer.h b/cpp/src/writer/tsfile_writer.h index e0b102c97..55e9e7f3a 100644 --- a/cpp/src/writer/tsfile_writer.h +++ b/cpp/src/writer/tsfile_writer.h @@ -84,6 +84,12 @@ class TsFileWriter { int write_tree(const TsRecord& record); int write_table(Tablet& tablet); + /** Add or replace a binary file-level property while the writer is open. */ + int add_tsfile_property(const std::string& key, const uint8_t* value, + uint32_t value_len); + int add_tsfile_property(const std::string& key, + const std::vector& value); + typedef std::map, MeasurementSchemaGroup*, IDeviceIDComparator> DeviceSchemasMap; diff --git a/cpp/test/common/tsfile_common_test.cc b/cpp/test/common/tsfile_common_test.cc index 2108b2d02..47f2b877e 100644 --- a/cpp/test/common/tsfile_common_test.cc +++ b/cpp/test/common/tsfile_common_test.cc @@ -449,9 +449,11 @@ TEST_F(TsFileMetaTest, SerializeDeserialize) { table_name, column_schemas, column_categories); meta_.table_schemas_.insert(std::make_pair(table_name, table_schema)); + meta_.tsfile_properties_.insert(std::make_pair( + "key", + TsFilePropertyValue(std::vector{'v', 'a', 'l', 'u', 'e'}))); meta_.tsfile_properties_.insert( - std::make_pair("key", new std::string("value"))); - meta_.tsfile_properties_.insert(std::make_pair("null_key", nullptr)); + std::make_pair("null_key", TsFilePropertyValue())); meta_.meta_offset_ = 456; void* buf = pa_.alloc(sizeof(BloomFilter)); @@ -471,8 +473,11 @@ TEST_F(TsFileMetaTest, SerializeDeserialize) { ASSERT_EQ(new_meta.table_schemas_.size(), 1); ASSERT_EQ( new_meta.table_schemas_[table_name]->get_column_categories().size(), 1); - ASSERT_EQ(*new_meta.tsfile_properties_["key"], std::string("value")); - ASSERT_EQ(new_meta.tsfile_properties_["null_key"], nullptr); + ASSERT_FALSE(new_meta.tsfile_properties_["key"].is_null); + ASSERT_EQ(new_meta.tsfile_properties_["key"].value, + (std::vector{'v', 'a', 'l', 'u', 'e'})); + ASSERT_TRUE(new_meta.tsfile_properties_["null_key"].is_null); + ASSERT_TRUE(new_meta.tsfile_properties_["null_key"].value.empty()); } // Regression: the default-compression configuration must name a compressor diff --git a/cpp/test/cwrapper/cwrapper_properties_test.cc b/cpp/test/cwrapper/cwrapper_properties_test.cc new file mode 100644 index 000000000..9167f7d7f --- /dev/null +++ b/cpp/test/cwrapper/cwrapper_properties_test.cc @@ -0,0 +1,166 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include + +#include +#include +#include + +#include "cwrapper/tsfile_cwrapper.h" +#include "utils/errno_define.h" + +namespace { + +const TsFileProperty* FindProperty(const TsFileProperty* properties, + uint32_t property_count, + const std::string& key) { + for (uint32_t i = 0; i < property_count; i++) { + if (properties[i].key_len == key.size() && + std::memcmp(properties[i].key, key.data(), key.size()) == 0) { + return &properties[i]; + } + } + return nullptr; +} + +TEST(CWrapperPropertiesTest, GenericWriterRoundTripsLengthAwareValues) { + const char* file_name = "cwrapper_properties_test.tsfile"; + std::remove(file_name); + + ERRNO error_code = common::E_OK; + TsFileWriter writer = + _tsfile_writer_new(file_name, 128 * 1024 * 1024, &error_code); + ASSERT_NE(nullptr, writer); + ASSERT_EQ(common::E_OK, error_code); + + const uint8_t first[] = {'f', 'i', 'r', 's', 't'}; + const uint8_t binary[] = {0x00, 0xFF, 0x80, 0x01, 0x00}; + const char embedded_null_key[] = {'k', '\0', 'y'}; + const uint8_t empty_marker = 0; + EXPECT_EQ(common::E_INVALID_ARG, + _tsfile_writer_add_tsfile_property(nullptr, "key", 3, binary, + sizeof(binary))); + EXPECT_EQ(common::E_INVALID_ARG, + _tsfile_writer_add_tsfile_property(writer, nullptr, 0, binary, + sizeof(binary))); + EXPECT_EQ(common::E_INVALID_ARG, + _tsfile_writer_add_tsfile_property(writer, "key", 3, nullptr, 1)); + ASSERT_EQ(common::E_OK, + _tsfile_writer_add_tsfile_property(writer, "overwritten", 11, + first, sizeof(first))); + ASSERT_EQ(common::E_OK, _tsfile_writer_flush(writer)); + ASSERT_EQ(common::E_OK, + _tsfile_writer_add_tsfile_property(writer, "overwritten", 11, + binary, sizeof(binary))); + ASSERT_EQ(common::E_OK, + _tsfile_writer_add_tsfile_property(writer, embedded_null_key, + sizeof(embedded_null_key), + binary, sizeof(binary))); + ASSERT_EQ(common::E_OK, _tsfile_writer_add_tsfile_property( + writer, "empty", 5, &empty_marker, 0)); + ASSERT_EQ(common::E_OK, _tsfile_writer_add_tsfile_property(writer, "null", + 4, nullptr, 0)); + ASSERT_EQ(common::E_OK, _tsfile_writer_close(writer)); + + TsFileReader reader = tsfile_reader_new(file_name, &error_code); + ASSERT_NE(nullptr, reader); + ASSERT_EQ(common::E_OK, error_code); + TsFileProperty* properties = nullptr; + uint32_t property_count = 0; + ASSERT_EQ(common::E_OK, tsfile_reader_get_tsfile_properties( + reader, &properties, &property_count)); + + const TsFileProperty* overwritten = + FindProperty(properties, property_count, "overwritten"); + ASSERT_NE(nullptr, overwritten); + EXPECT_FALSE(overwritten->is_null); + ASSERT_EQ(sizeof(binary), overwritten->value_len); + EXPECT_EQ(0, std::memcmp(binary, overwritten->value, sizeof(binary))); + + const TsFileProperty* embedded_key_property = + FindProperty(properties, property_count, + std::string(embedded_null_key, sizeof(embedded_null_key))); + ASSERT_NE(nullptr, embedded_key_property); + ASSERT_EQ(sizeof(binary), embedded_key_property->value_len); + EXPECT_EQ( + 0, std::memcmp(binary, embedded_key_property->value, sizeof(binary))); + + const TsFileProperty* empty = + FindProperty(properties, property_count, "empty"); + ASSERT_NE(nullptr, empty); + EXPECT_FALSE(empty->is_null); + EXPECT_EQ(0U, empty->value_len); + + const TsFileProperty* null_value = + FindProperty(properties, property_count, "null"); + ASSERT_NE(nullptr, null_value); + EXPECT_TRUE(null_value->is_null); + EXPECT_EQ(0U, null_value->value_len); + + tsfile_free_tsfile_properties(properties, property_count); + TsFileProperty sentinel{}; + properties = &sentinel; + property_count = 1; + EXPECT_EQ(common::E_INVALID_ARG, + tsfile_reader_get_tsfile_properties(nullptr, &properties, + &property_count)); + EXPECT_EQ(nullptr, properties); + EXPECT_EQ(0U, property_count); + EXPECT_EQ(common::E_OK, tsfile_reader_close(reader)); + EXPECT_EQ(0, std::remove(file_name)); +} + +TEST(CWrapperPropertiesTest, TableWriterSetterUsesExplicitLengths) { + const char* file_name = "cwrapper_table_properties_test.tsfile"; + std::remove(file_name); + + ERRNO error_code = common::E_OK; + WriteFile file = write_file_new(file_name, &error_code); + ASSERT_NE(nullptr, file); + ASSERT_EQ(common::E_OK, error_code); + + ColumnSchema column = {const_cast("value"), TS_DATATYPE_INT64, + FIELD}; + TableSchema schema = {const_cast("table"), &column, 1}; + TsFileWriter writer = tsfile_writer_new(file, &schema, &error_code); + ASSERT_NE(nullptr, writer); + const uint8_t binary[] = {0xAA, 0x00, 0xBB}; + ASSERT_EQ(common::E_OK, tsfile_writer_add_tsfile_property( + writer, "binary", 6, binary, sizeof(binary))); + ASSERT_EQ(common::E_OK, tsfile_writer_close(writer)); + free_write_file(&file); + + TsFileReader reader = tsfile_reader_new(file_name, &error_code); + ASSERT_NE(nullptr, reader); + TsFileProperty* properties = nullptr; + uint32_t property_count = 0; + ASSERT_EQ(common::E_OK, tsfile_reader_get_tsfile_properties( + reader, &properties, &property_count)); + const TsFileProperty* property = + FindProperty(properties, property_count, "binary"); + ASSERT_NE(nullptr, property); + ASSERT_EQ(sizeof(binary), property->value_len); + EXPECT_EQ(0, std::memcmp(binary, property->value, sizeof(binary))); + tsfile_free_tsfile_properties(properties, property_count); + EXPECT_EQ(common::E_OK, tsfile_reader_close(reader)); + EXPECT_EQ(0, std::remove(file_name)); +} + +} // namespace diff --git a/cpp/test/writer/table_view/tsfile_writer_table_test.cc b/cpp/test/writer/table_view/tsfile_writer_table_test.cc index 0dfaccc06..2dd9b5643 100644 --- a/cpp/test/writer/table_view/tsfile_writer_table_test.cc +++ b/cpp/test/writer/table_view/tsfile_writer_table_test.cc @@ -144,6 +144,30 @@ TEST_F(TsFileWriterTableTest, WriteTableTest) { delete table_schema; } +TEST_F(TsFileWriterTableTest, AddTsFilePropertyDelegatesToWriter) { + auto table_schema = gen_table_schema(0); + TsFileTableWriter writer(&write_file_, table_schema); + const std::vector before_flush = {'b', 'e', 'f', 'o', 'r', 'e'}; + const std::vector after_flush = {0x00, 0xFF, 0x01}; + + ASSERT_EQ(common::E_OK, + writer.add_tsfile_property("table-property", before_flush)); + ASSERT_EQ(common::E_OK, writer.flush()); + ASSERT_EQ(common::E_OK, + writer.add_tsfile_property("table-property", after_flush)); + ASSERT_EQ(common::E_OK, writer.close()); + ASSERT_EQ(common::E_FILE_WRITE_ERR, + writer.add_tsfile_property("closed", after_flush)); + + TsFileReader reader; + ASSERT_EQ(common::E_OK, reader.open(file_name_)); + TsFileProperties properties = reader.get_tsfile_properties(); + ASSERT_FALSE(properties.at("table-property").is_null); + EXPECT_EQ(after_flush, properties.at("table-property").value); + EXPECT_EQ(common::E_OK, reader.close()); + delete table_schema; +} + TEST_F(TsFileWriterTableTest, WithoutTagAndMultiPage) { std::vector measurement_schemas; std::vector column_categories; diff --git a/cpp/test/writer/tsfile_properties_test.cc b/cpp/test/writer/tsfile_properties_test.cc new file mode 100644 index 000000000..4590c1bcd --- /dev/null +++ b/cpp/test/writer/tsfile_properties_test.cc @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include + +#include +#include + +#include "common/global.h" +#include "reader/tsfile_reader.h" +#include "writer/tsfile_writer.h" + +namespace storage { + +namespace { + +std::vector Bytes(const std::string& value) { + return std::vector(value.begin(), value.end()); +} + +class TsFilePropertiesTest : public ::testing::Test { + protected: + void SetUp() override { + libtsfile_init(); + file_name_ = "tsfile_properties_test.tsfile"; + std::remove(file_name_.c_str()); + } + + void TearDown() override { + std::remove(file_name_.c_str()); + libtsfile_destroy(); + } + + std::string file_name_; +}; + +TEST_F(TsFilePropertiesTest, WriterPreservesBinaryNullAndEmptyValues) { + TsFileWriter writer; + ASSERT_EQ(common::E_OK, writer.open(file_name_)); + + const std::vector first_value = {'f', 'i', 'r', 's', 't'}; + ASSERT_EQ(common::E_OK, + writer.add_tsfile_property("overwritten", first_value)); + ASSERT_EQ(common::E_OK, writer.flush()); + + const std::vector binary_value = {0x00, 0x7F, 0x80, 0xFF, 0x00}; + ASSERT_EQ(common::E_OK, + writer.add_tsfile_property("overwritten", binary_value)); + std::vector copied_value = {0x10, 0x20, 0x30}; + ASSERT_EQ(common::E_OK, writer.add_tsfile_property("copied", copied_value)); + copied_value[0] = 0xFF; + const std::string embedded_null_key("embedded\0key", 12); + ASSERT_EQ(common::E_OK, + writer.add_tsfile_property(embedded_null_key, binary_value)); + ASSERT_EQ(common::E_OK, + writer.add_tsfile_property("empty", std::vector())); + ASSERT_EQ(common::E_OK, writer.add_tsfile_property("null", nullptr, 0)); + ASSERT_EQ(common::E_INVALID_ARG, + writer.add_tsfile_property("invalid", nullptr, 1)); + ASSERT_EQ(common::E_OK, + writer.add_tsfile_property("encryptLevel", Bytes("custom"))); + ASSERT_EQ(common::E_OK, + writer.add_tsfile_property("encryptType", Bytes("custom"))); + ASSERT_EQ(common::E_OK, + writer.add_tsfile_property("encryptKey", Bytes("custom"))); + ASSERT_EQ(common::E_OK, writer.close()); + ASSERT_EQ(common::E_FILE_WRITE_ERR, + writer.add_tsfile_property("closed", binary_value)); + + TsFileReader reader; + ASSERT_EQ(common::E_OK, reader.open(file_name_)); + TsFileProperties properties = reader.get_tsfile_properties(); + + ASSERT_FALSE(properties.at("overwritten").is_null); + EXPECT_EQ(binary_value, properties.at("overwritten").value); + ASSERT_FALSE(properties.at("empty").is_null); + EXPECT_TRUE(properties.at("empty").value.empty()); + EXPECT_EQ((std::vector{0x10, 0x20, 0x30}), + properties.at("copied").value); + EXPECT_EQ(binary_value, properties.at(embedded_null_key).value); + EXPECT_TRUE(properties.at("null").is_null); + EXPECT_TRUE(properties.at("null").value.empty()); + + ASSERT_FALSE(properties.at("encryptLevel").is_null); + EXPECT_EQ(Bytes("0"), properties.at("encryptLevel").value); + ASSERT_FALSE(properties.at("encryptType").is_null); + EXPECT_EQ(Bytes("org.apache.tsfile.encrypt.UNENCRYPTED"), + properties.at("encryptType").value); + EXPECT_TRUE(properties.at("encryptKey").is_null); + EXPECT_TRUE(properties.at("encryptKey").value.empty()); + EXPECT_EQ(common::E_OK, reader.close()); +} + +} // namespace + +} // namespace storage diff --git a/python/README-zh.md b/python/README-zh.md index 660c001e8..dd3a59ea4 100644 --- a/python/README-zh.md +++ b/python/README-zh.md @@ -66,4 +66,20 @@ mvn -P with-cpp,with-python clean verify ```sh python setup.py build_ext --inplace -``` \ No newline at end of file +``` + +## 文件级 Properties + +`TsFileWriter` 和 `TsFileTableWriter` 可以在打开期间写入二进制 property。 +setter 仅接受 `bytes`。reader 返回 `dict[str, bytes | None]`，并区分 null 与 +零长度 bytes。 + +```python +with TsFileWriter("example.tsfile") as writer: + writer.add_tsfile_property("binary-property", b"\x01\x00\xff") + +with TsFileReader("example.tsfile") as reader: + properties = reader.get_tsfile_properties() +``` + +Property value 不携带数据类型；保存数字或结构体时应使用明确、可跨语言的字节编码。 diff --git a/python/README.md b/python/README.md index 51cb498ec..8e2716a2c 100644 --- a/python/README.md +++ b/python/README.md @@ -61,3 +61,19 @@ Build by python command: python setup.py build_ext --inplace ``` +## File-level properties + +`TsFileWriter` and `TsFileTableWriter` accept binary properties while they are +open. The setter accepts `bytes` only. Readers return `dict[str, bytes | None]`, +preserving null and empty values separately. + +```python +with TsFileWriter("example.tsfile") as writer: + writer.add_tsfile_property("binary-property", b"\x01\x00\xff") + +with TsFileReader("example.tsfile") as reader: + properties = reader.get_tsfile_properties() +``` + +Values do not carry a data type; use an explicit portable encoding when storing +numbers or structures. diff --git a/python/tests/test_tsfile_properties.py b/python/tests/test_tsfile_properties.py new file mode 100644 index 000000000..63621ce8a --- /dev/null +++ b/python/tests/test_tsfile_properties.py @@ -0,0 +1,79 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import os + +import pytest + +from tsfile import ( + ColumnCategory, + ColumnSchema, + FileWriteError, + TableSchema, + TSDataType, + TsFileReader, + TsFileTableWriter, + TsFileWriter, +) + + +def test_tsfile_writer_properties_round_trip(tmp_path): + path = os.fspath(tmp_path / "writer-properties.tsfile") + writer = TsFileWriter(path) + writer.add_tsfile_property("overwritten", b"first") + writer.flush() + writer.add_tsfile_property("overwritten", b"\x00\xff\x80\x00") + writer.add_tsfile_property("empty", b"") + writer.add_tsfile_property("embedded\x00key", b"binary-key") + + with pytest.raises(TypeError): + writer.add_tsfile_property("text", "not-bytes") + with pytest.raises(TypeError): + writer.add_tsfile_property("bytearray", bytearray(b"not-bytes")) + with pytest.raises(TypeError): + writer.add_tsfile_property("bytes-subclass", type("B", (bytes,), {})(b"value")) + + writer.close() + with pytest.raises(FileWriteError): + writer.add_tsfile_property("closed", b"value") + + with TsFileReader(path) as reader: + properties = reader.get_tsfile_properties() + assert properties["overwritten"] == b"\x00\xff\x80\x00" + assert properties["empty"] == b"" + assert properties["embedded\x00key"] == b"binary-key" + assert properties["encryptLevel"] == b"0" + assert properties["encryptType"] == b"org.apache.tsfile.encrypt.UNENCRYPTED" + assert properties["encryptKey"] is None + + +def test_tsfile_table_writer_property_delegation(tmp_path): + path = os.fspath(tmp_path / "table-writer-properties.tsfile") + schema = TableSchema( + "table", + [ColumnSchema("value", TSDataType.INT64, ColumnCategory.FIELD)], + ) + writer = TsFileTableWriter(path, schema) + writer.add_tsfile_property("table-property", b"before") + writer.flush() + writer.add_tsfile_property("table-property", b"after\x00\xff") + writer.close() + with pytest.raises(FileWriteError): + writer.add_tsfile_property("closed", b"value") + + with TsFileReader(path) as reader: + assert reader.get_tsfile_properties()["table-property"] == b"after\x00\xff" diff --git a/python/tsfile/tsfile_cpp.pxd b/python/tsfile/tsfile_cpp.pxd index cc14f4034..4daf0b153 100644 --- a/python/tsfile/tsfile_cpp.pxd +++ b/python/tsfile/tsfile_cpp.pxd @@ -25,6 +25,7 @@ cdef extern from "cwrapper/errno_define_c.h": enum: RET_OK RET_NO_MORE_DATA + RET_FILE_WRITE_ERR # import symbols from tsfile_cwrapper.h cdef extern from "cwrapper/tsfile_cwrapper.h": @@ -181,6 +182,13 @@ cdef extern from "cwrapper/tsfile_cwrapper.h": DeviceTimeseriesMetadataEntry * entries uint32_t device_count + ctypedef struct TsFileProperty: + char * key + uint32_t key_len + uint8_t * value + uint32_t value_len + bint is_null + ctypedef struct ResultSetMetaData: char** column_names TSDataType * data_types @@ -201,6 +209,9 @@ cdef extern from "cwrapper/tsfile_cwrapper.h": # writer : flush ErrorCode _tsfile_writer_flush(TsFileWriter writer); + ErrorCode _tsfile_writer_add_tsfile_property( + TsFileWriter writer, const char * key, uint32_t key_len, + const uint8_t * value, uint32_t value_len); # writer : register table, device and timeseries ErrorCode _tsfile_writer_register_table(TsFileWriter writer, TableSchema * schema); @@ -316,6 +327,11 @@ cdef extern from "cwrapper/tsfile_cwrapper.h": DeviceTimeseriesMetadataMap * out_map); void tsfile_free_device_timeseries_metadata_map( DeviceTimeseriesMetadataMap * map); + ErrorCode tsfile_reader_get_tsfile_properties( + TsFileReader reader, TsFileProperty ** out_properties, + uint32_t * out_length); + void tsfile_free_tsfile_properties(TsFileProperty * properties, + uint32_t length); # Tag filter types and functions diff --git a/python/tsfile/tsfile_reader.pyx b/python/tsfile/tsfile_reader.pyx index 36374adde..a9b891fa2 100644 --- a/python/tsfile/tsfile_reader.pyx +++ b/python/tsfile/tsfile_reader.pyx @@ -26,7 +26,7 @@ from libc.string cimport strlen from cpython.bytes cimport PyBytes_FromStringAndSize from libc.string cimport memset import pyarrow as pa -from libc.stdint cimport INT64_MIN, INT64_MAX, uintptr_t +from libc.stdint cimport INT64_MIN, INT64_MAX, uint32_t, uintptr_t from tsfile.schema import TSDataType as TSDataTypePy from tsfile.schema import DeviceID, DeviceTimeseriesMetadataGroup @@ -518,6 +518,40 @@ cdef class TsFileReaderPy: """ return reader_get_timeseries_metadata_c(self.reader, device_ids) + def get_tsfile_properties(self) -> Dict[str, Optional[bytes]]: + """ + Return file-level properties as ``dict[str, bytes | None]``. + + Null property values are returned as ``None`` and remain distinct from + non-null zero-length byte strings. + """ + cdef TsFileProperty * properties = NULL + cdef uint32_t property_count = 0 + cdef uint32_t i + cdef ErrorCode err_code + cdef object key + cdef dict result = {} + + err_code = tsfile_reader_get_tsfile_properties( + self.reader, &properties, &property_count + ) + check_error(err_code) + try: + for i in range(property_count): + key = PyBytes_FromStringAndSize( + properties[i].key, properties[i].key_len + ).decode('utf-8') + if properties[i].is_null: + result[key] = None + else: + result[key] = PyBytes_FromStringAndSize( + properties[i].value, + properties[i].value_len, + ) + finally: + tsfile_free_tsfile_properties(properties, property_count) + return result + def close(self): """ Close TsFile Reader, if reader has result sets, invalid them. diff --git a/python/tsfile/tsfile_table_writer.py b/python/tsfile/tsfile_table_writer.py index 9f3a257e6..e4fceef97 100644 --- a/python/tsfile/tsfile_table_writer.py +++ b/python/tsfile/tsfile_table_writer.py @@ -232,6 +232,12 @@ def close(self): """ self.writer.close() + def add_tsfile_property(self, key: str, value: bytes): + """ + Add or replace a binary file-level property while the writer is open. + """ + self.writer.add_tsfile_property(key, value) + def flush(self): """ Flush current data to tsfile. diff --git a/python/tsfile/tsfile_writer.pyx b/python/tsfile/tsfile_writer.pyx index 9e84d83c0..4df03b449 100644 --- a/python/tsfile/tsfile_writer.pyx +++ b/python/tsfile/tsfile_writer.pyx @@ -23,7 +23,8 @@ from tsfile.schema import TableSchema as TableSchemaPy from tsfile.schema import TimeseriesSchema as TimeseriesSchemaPy, DeviceSchema as DeviceSchemaPy from tsfile.tablet import Tablet as TabletPy from libc.string cimport memset -from libc.stdint cimport uintptr_t +from libc.stdint cimport uint32_t, uint8_t, uintptr_t +from cpython.bytes cimport PyBytes_AsStringAndSize from .tsfile_cpp cimport * from .tsfile_py_cpp cimport * @@ -163,6 +164,40 @@ cdef class TsFileWriterPy: if arrow_schema.release != NULL: arrow_schema.release(&arrow_schema) + def add_tsfile_property(self, key: str, value: bytes): + """ + Add or replace a binary file-level property while the writer is open. + + ``value`` must be ``bytes``. The data is copied immediately, and a + later call with the same key replaces the previous value. + """ + if not isinstance(key, str): + raise TypeError("TsFile property key must be str") + if type(value) is not bytes: + raise TypeError("TsFile property value must be bytes") + if self.writer == NULL: + check_error(RET_FILE_WRITE_ERR, b"TsFile writer is closed") + + cdef bytes encoded_key = key.encode('utf-8') + cdef char * value_ptr = NULL + cdef Py_ssize_t value_len = 0 + cdef ErrorCode errno + if len(encoded_key) > 0x7FFFFFFF: + raise OverflowError("TsFile property key is too large") + if PyBytes_AsStringAndSize(value, &value_ptr, &value_len) < 0: + raise TypeError("TsFile property value must be bytes") + if value_len > 0x7FFFFFFF: + raise OverflowError("TsFile property value is too large") + + errno = _tsfile_writer_add_tsfile_property( + self.writer, + encoded_key, + len(encoded_key), + value_ptr, + value_len, + ) + check_error(errno) + cpdef close(self): """ Flush data and Close tsfile writer. From 7a13dedfde04844feb0594ba7b65507a40c8363f Mon Sep 17 00:00:00 2001 From: ColinLee Date: Wed, 5 Aug 2026 17:55:11 +0800 Subject: [PATCH 2/4] Fix TsFile properties metadata compatibility --- cpp/src/common/tsfile_common.cc | 103 ++++++++++++++---- cpp/src/common/tsfile_common.h | 2 +- cpp/src/cwrapper/tsfile_cwrapper.cc | 8 +- cpp/src/file/tsfile_io_writer.cc | 15 ++- cpp/src/reader/bloom_filter.cc | 20 +++- cpp/test/common/tsfile_common_test.cc | 38 ++++++- cpp/test/cwrapper/cwrapper_properties_test.cc | 11 ++ cpp/test/reader/bloom_filter_test.cc | 37 +++++++ python/tests/test_tsfile_properties.py | 19 ++++ python/tsfile/tsfile_reader.pyx | 12 +- 10 files changed, 227 insertions(+), 38 deletions(-) diff --git a/cpp/src/common/tsfile_common.cc b/cpp/src/common/tsfile_common.cc index f0d97dc00..3cac258ff 100644 --- a/cpp/src/common/tsfile_common.cc +++ b/cpp/src/common/tsfile_common.cc @@ -20,6 +20,7 @@ #include "common/tsfile_common.h" #include +#include #include #include "common/logger/elog.h" @@ -180,45 +181,101 @@ int TSMIterator::get_next(std::shared_ptr& ret_device_id, return ret; } -int TsFileMeta::serialize_to(common::ByteStream& out) { +int TsFileMeta::serialize_to(common::ByteStream& out, + int32_t& serialized_size) { + serialized_size = 0; + const size_t max_property_size = + static_cast(std::numeric_limits::max()); + if (tsfile_properties_.size() > max_property_size) { + return common::E_OUT_OF_RANGE; + } + for (const auto& tsfile_property : tsfile_properties_) { + if (tsfile_property.first.size() > max_property_size || + (!tsfile_property.second.is_null && + tsfile_property.second.value.size() > max_property_size)) { + return common::E_OUT_OF_RANGE; + } + } + + int ret = common::E_OK; auto start_idx = out.total_size(); - common::SerializationUtil::write_var_uint( - table_metadata_index_node_map_.size(), out); + if (RET_FAIL(common::SerializationUtil::write_var_uint( + table_metadata_index_node_map_.size(), out))) { + return ret; + } for (auto& idx_nodes_iter : table_metadata_index_node_map_) { - common::SerializationUtil::write_var_str(idx_nodes_iter.first, out); - idx_nodes_iter.second->serialize_to(out); + if (RET_FAIL(common::SerializationUtil::write_var_str( + idx_nodes_iter.first, out))) { + return ret; + } else if (RET_FAIL(idx_nodes_iter.second->serialize_to(out))) { + return ret; + } } - common::SerializationUtil::write_var_uint(table_schemas_.size(), out); + if (RET_FAIL(common::SerializationUtil::write_var_uint( + table_schemas_.size(), out))) { + return ret; + } for (auto& table_schema_iter : table_schemas_) { - common::SerializationUtil::write_var_str(table_schema_iter.first, out); - table_schema_iter.second->serialize_to(out); + if (RET_FAIL(common::SerializationUtil::write_var_str( + table_schema_iter.first, out))) { + return ret; + } else if (RET_FAIL(table_schema_iter.second->serialize_to(out))) { + return ret; + } } - common::SerializationUtil::write_i64(meta_offset_, out); + if (RET_FAIL(common::SerializationUtil::write_i64(meta_offset_, out))) { + return ret; + } if (bloom_filter_ != nullptr) { - bloom_filter_->serialize_to(out); + if (RET_FAIL(bloom_filter_->serialize_to(out))) { + return ret; + } } else { - common::SerializationUtil::write_ui8(0, out); + if (RET_FAIL(common::SerializationUtil::write_ui8(0, out))) { + return ret; + } } - common::SerializationUtil::write_var_int(tsfile_properties_.size(), out); + if (RET_FAIL(common::SerializationUtil::write_var_int( + static_cast(tsfile_properties_.size()), out))) { + return ret; + } for (const auto& tsfile_property : tsfile_properties_) { - common::SerializationUtil::write_var_str(tsfile_property.first, out); + if (RET_FAIL(common::SerializationUtil::write_var_str( + tsfile_property.first, out))) { + return ret; + } const TsFilePropertyValue& value = tsfile_property.second; if (value.is_null) { - common::SerializationUtil::write_var_int(NO_STR_TO_READ, out); + if (RET_FAIL(common::SerializationUtil::write_var_int( + NO_STR_TO_READ, out))) { + return ret; + } } else { - common::SerializationUtil::write_var_int( - static_cast(value.value.size()), out); + if (RET_FAIL(common::SerializationUtil::write_var_int( + static_cast(value.value.size()), out))) { + return ret; + } if (!value.value.empty()) { - out.write_buf(value.value.data(), value.value.size()); + if (RET_FAIL(out.write_buf( + value.value.data(), + static_cast(value.value.size())))) { + return ret; + } } } } - return out.total_size() - start_idx; + const uint64_t total_size = out.total_size() - start_idx; + if (total_size > + static_cast(std::numeric_limits::max())) { + return common::E_OUT_OF_RANGE; + } + serialized_size = static_cast(total_size); + return common::E_OK; } int TsFileMeta::deserialize_from(common::ByteStream& in) { @@ -259,7 +316,9 @@ int TsFileMeta::deserialize_from(common::ByteStream& in) { common::SerializationUtil::read_i64(meta_offset_, in); - bloom_filter_->deserialize_from(in); + if (RET_FAIL(bloom_filter_->deserialize_from(in))) { + return ret; + } int32_t tsfile_properties_size = 0; if (RET_FAIL(common::SerializationUtil::read_var_int(tsfile_properties_size, @@ -278,6 +337,9 @@ int TsFileMeta::deserialize_from(common::ByteStream& in) { } else if (key_len < 0) { return common::E_TSFILE_CORRUPTED; } + if (static_cast(key_len) > in.remaining_size()) { + return common::E_TSFILE_CORRUPTED; + } key.resize(static_cast(key_len)); if (key_len > 0) { uint32_t read_len = 0; @@ -299,6 +361,9 @@ int TsFileMeta::deserialize_from(common::ByteStream& in) { } else if (value_len < 0) { return common::E_TSFILE_CORRUPTED; } else { + if (static_cast(value_len) > in.remaining_size()) { + return common::E_TSFILE_CORRUPTED; + } value.is_null = false; value.value.resize(static_cast(value_len)); if (value_len > 0) { diff --git a/cpp/src/common/tsfile_common.h b/cpp/src/common/tsfile_common.h index 811a11f22..d763acbbd 100644 --- a/cpp/src/common/tsfile_common.h +++ b/cpp/src/common/tsfile_common.h @@ -1197,7 +1197,7 @@ struct TsFileMeta { table_schemas_.clear(); } - int serialize_to(common::ByteStream& out); + int serialize_to(common::ByteStream& out, int32_t& serialized_size); int deserialize_from(common::ByteStream& in); diff --git a/cpp/src/cwrapper/tsfile_cwrapper.cc b/cpp/src/cwrapper/tsfile_cwrapper.cc index a969cfc9a..3e27f60d0 100644 --- a/cpp/src/cwrapper/tsfile_cwrapper.cc +++ b/cpp/src/cwrapper/tsfile_cwrapper.cc @@ -267,7 +267,9 @@ ERRNO tsfile_writer_add_tsfile_property(TsFileWriter writer, const char* key, (value == nullptr && value_len > 0)) { return common::E_INVALID_ARG; } - if (key_len > static_cast(std::numeric_limits::max())) { + if (key_len > static_cast(std::numeric_limits::max()) || + value_len > + static_cast(std::numeric_limits::max())) { return common::E_OUT_OF_RANGE; } try { @@ -1772,7 +1774,9 @@ ERRNO _tsfile_writer_add_tsfile_property(TsFileWriter writer, const char* key, (value == nullptr && value_len > 0)) { return common::E_INVALID_ARG; } - if (key_len > static_cast(std::numeric_limits::max())) { + if (key_len > static_cast(std::numeric_limits::max()) || + value_len > + static_cast(std::numeric_limits::max())) { return common::E_OUT_OF_RANGE; } try { diff --git a/cpp/src/file/tsfile_io_writer.cc b/cpp/src/file/tsfile_io_writer.cc index 37221dd95..29ddf0d90 100644 --- a/cpp/src/file/tsfile_io_writer.cc +++ b/cpp/src/file/tsfile_io_writer.cc @@ -125,14 +125,14 @@ int TsFileIOWriter::add_tsfile_property(const std::string& key, int TsFileIOWriter::add_tsfile_property(const std::string& key, const std::vector& value) { + if (file_ == nullptr || file_->get_fd() < 0) { + return common::E_FILE_WRITE_ERR; + } if (key.size() > static_cast(std::numeric_limits::max()) || value.size() > static_cast(std::numeric_limits::max())) { return common::E_OUT_OF_RANGE; } - if (file_ == nullptr || file_->get_fd() < 0) { - return common::E_FILE_WRITE_ERR; - } tsfile_properties_[key] = TsFilePropertyValue(value); return common::E_OK; } @@ -517,9 +517,12 @@ int TsFileIOWriter::write_file_index() { #if DEBUG_SE auto tsfile_meta_offset = write_stream_.total_size(); #endif - auto total_write_size = tsfile_meta.serialize_to(write_stream_); - if (RET_FAIL(common::SerializationUtil::write_i32(total_write_size, - write_stream_))) { + int32_t total_write_size = 0; + if (RET_FAIL( + tsfile_meta.serialize_to(write_stream_, total_write_size))) { + return ret; + } else if (RET_FAIL(common::SerializationUtil::write_i32( + total_write_size, write_stream_))) { return ret; } tsfile_meta.bloom_filter_ = nullptr; diff --git a/cpp/src/reader/bloom_filter.cc b/cpp/src/reader/bloom_filter.cc index 4aff4ecd3..acba67f27 100644 --- a/cpp/src/reader/bloom_filter.cc +++ b/cpp/src/reader/bloom_filter.cc @@ -235,11 +235,12 @@ int BloomFilter::serialize_to(ByteStream& out) { bitset_.to_bytes(filter_data_bytes, filter_data_bytes_len); if (RET_FAIL( SerializationUtil::write_var_uint(filter_data_bytes_len, out))) { - } else if (RET_FAIL( - out.write_buf(filter_data_bytes, filter_data_bytes_len))) { - } else if (RET_FAIL(SerializationUtil::write_var_uint(size_, out))) { - } else if (RET_FAIL( - SerializationUtil::write_var_uint(hash_func_count_, out))) { + } else if (filter_data_bytes_len > 0) { + if (RET_FAIL(out.write_buf(filter_data_bytes, filter_data_bytes_len))) { + } else if (RET_FAIL(SerializationUtil::write_var_uint(size_, out))) { + } else if (RET_FAIL(SerializationUtil::write_var_uint(hash_func_count_, + out))) { + } } if (filter_data_bytes_len > 0) { bitset_.revert_bytes(filter_data_bytes); @@ -253,6 +254,12 @@ int BloomFilter::deserialize_from(ByteStream& in) { uint32_t ret_read_len = 0; uint8_t* filter_data = nullptr; if (RET_FAIL(SerializationUtil::read_var_uint(filter_data_bytes_len, in))) { + } else if (filter_data_bytes_len == 0) { + size_ = 0; + hash_func_count_ = 0; + return E_OK; + } else if (filter_data_bytes_len > in.remaining_size()) { + ret = E_TSFILE_CORRUPTED; } else if (UNLIKELY(nullptr == (filter_data = (uint8_t*)mem_alloc( filter_data_bytes_len, MOD_BLOOM_FILTER)))) { @@ -264,6 +271,9 @@ int BloomFilter::deserialize_from(ByteStream& in) { } else if (RET_FAIL(SerializationUtil::read_var_uint(size_, in))) { } else if (RET_FAIL( SerializationUtil::read_var_uint(hash_func_count_, in))) { + } else if (size_ == 0 || hash_func_count_ == 0 || + hash_func_count_ > MAX_HASH_FUNC_COUNT) { + ret = E_TSFILE_CORRUPTED; } else { for (uint32_t i = 0; i < hash_func_count_; i++) { hash_func_arr_[i].init(size_, SEEDS[i]); diff --git a/cpp/test/common/tsfile_common_test.cc b/cpp/test/common/tsfile_common_test.cc index 47f2b877e..fcadaa440 100644 --- a/cpp/test/common/tsfile_common_test.cc +++ b/cpp/test/common/tsfile_common_test.cc @@ -460,10 +460,12 @@ TEST_F(TsFileMetaTest, SerializeDeserialize) { meta_.bloom_filter_ = new (buf) BloomFilter(); meta_.bloom_filter_->init(0.1, 100); - meta_.serialize_to(*out_); + int32_t serialized_size = 0; + ASSERT_EQ(common::E_OK, meta_.serialize_to(*out_, serialized_size)); + ASSERT_EQ(serialized_size, out_->total_size()); TsFileMeta new_meta(&pa_); - new_meta.deserialize_from(*out_); + ASSERT_EQ(common::E_OK, new_meta.deserialize_from(*out_)); ASSERT_EQ(new_meta.meta_offset_, 456); ASSERT_EQ(new_meta.table_metadata_index_node_map_.size(), 1); @@ -480,6 +482,38 @@ TEST_F(TsFileMetaTest, SerializeDeserialize) { ASSERT_TRUE(new_meta.tsfile_properties_["null_key"].value.empty()); } +TEST_F(TsFileMetaTest, RejectsPropertyKeyLengthBeyondRemainingInput) { + ASSERT_EQ(common::E_OK, + common::SerializationUtil::write_var_uint(0, *out_)); + ASSERT_EQ(common::E_OK, + common::SerializationUtil::write_var_uint(0, *out_)); + ASSERT_EQ(common::E_OK, common::SerializationUtil::write_i64(0, *out_)); + ASSERT_EQ(common::E_OK, common::SerializationUtil::write_ui8(0, *out_)); + ASSERT_EQ(common::E_OK, common::SerializationUtil::write_var_int(1, *out_)); + ASSERT_EQ(common::E_OK, + common::SerializationUtil::write_var_int(1024, *out_)); + + TsFileMeta meta(&pa_); + EXPECT_EQ(common::E_TSFILE_CORRUPTED, meta.deserialize_from(*out_)); +} + +TEST_F(TsFileMetaTest, RejectsPropertyValueLengthBeyondRemainingInput) { + ASSERT_EQ(common::E_OK, + common::SerializationUtil::write_var_uint(0, *out_)); + ASSERT_EQ(common::E_OK, + common::SerializationUtil::write_var_uint(0, *out_)); + ASSERT_EQ(common::E_OK, common::SerializationUtil::write_i64(0, *out_)); + ASSERT_EQ(common::E_OK, common::SerializationUtil::write_ui8(0, *out_)); + ASSERT_EQ(common::E_OK, common::SerializationUtil::write_var_int(1, *out_)); + ASSERT_EQ(common::E_OK, common::SerializationUtil::write_var_int(1, *out_)); + ASSERT_EQ(common::E_OK, out_->write_buf("k", 1)); + ASSERT_EQ(common::E_OK, + common::SerializationUtil::write_var_int(1024, *out_)); + + TsFileMeta meta(&pa_); + EXPECT_EQ(common::E_TSFILE_CORRUPTED, meta.deserialize_from(*out_)); +} + // Regression: the default-compression configuration must name a compressor // that the build actually provides; otherwise CompressorFactory returns // nullptr at write time. init_config_value() previously gated SNAPPY on diff --git a/cpp/test/cwrapper/cwrapper_properties_test.cc b/cpp/test/cwrapper/cwrapper_properties_test.cc index 9167f7d7f..7b2e43116 100644 --- a/cpp/test/cwrapper/cwrapper_properties_test.cc +++ b/cpp/test/cwrapper/cwrapper_properties_test.cc @@ -21,6 +21,7 @@ #include #include +#include #include #include "cwrapper/tsfile_cwrapper.h" @@ -62,6 +63,11 @@ TEST(CWrapperPropertiesTest, GenericWriterRoundTripsLengthAwareValues) { sizeof(binary))); EXPECT_EQ(common::E_INVALID_ARG, _tsfile_writer_add_tsfile_property(writer, "key", 3, nullptr, 1)); + const uint32_t oversized_len = + static_cast(std::numeric_limits::max()) + 1U; + EXPECT_EQ(common::E_OUT_OF_RANGE, + _tsfile_writer_add_tsfile_property(writer, "key", 3, binary, + oversized_len)); ASSERT_EQ(common::E_OK, _tsfile_writer_add_tsfile_property(writer, "overwritten", 11, first, sizeof(first))); @@ -142,6 +148,11 @@ TEST(CWrapperPropertiesTest, TableWriterSetterUsesExplicitLengths) { TsFileWriter writer = tsfile_writer_new(file, &schema, &error_code); ASSERT_NE(nullptr, writer); const uint8_t binary[] = {0xAA, 0x00, 0xBB}; + const uint32_t oversized_len = + static_cast(std::numeric_limits::max()) + 1U; + EXPECT_EQ(common::E_OUT_OF_RANGE, + tsfile_writer_add_tsfile_property(writer, "binary", 6, binary, + oversized_len)); ASSERT_EQ(common::E_OK, tsfile_writer_add_tsfile_property( writer, "binary", 6, binary, sizeof(binary))); ASSERT_EQ(common::E_OK, tsfile_writer_close(writer)); diff --git a/cpp/test/reader/bloom_filter_test.cc b/cpp/test/reader/bloom_filter_test.cc index 29b24db97..71ceb643e 100644 --- a/cpp/test/reader/bloom_filter_test.cc +++ b/cpp/test/reader/bloom_filter_test.cc @@ -64,3 +64,40 @@ TEST(BloomfilterTest, BloomFilter) { common::mem_free(filter_data_bytes); common::mem_free(filter_data_bytes2); } + +TEST(BloomfilterTest, EmptyFilterUsesJavaCompatibleEncoding) { + BloomFilter filter; + ASSERT_EQ(common::E_OK, filter.init(0.1, 0)); + + common::ByteStream out(1024, common::MOD_DEFAULT); + ASSERT_EQ(common::E_OK, filter.serialize_to(out)); + ASSERT_EQ(1U, out.total_size()); + + BloomFilter deserialized; + ASSERT_EQ(common::E_OK, deserialized.deserialize_from(out)); + EXPECT_TRUE(deserialized.is_empty()); + EXPECT_EQ(0U, out.remaining_size()); +} + +TEST(BloomfilterTest, RejectsInvalidHashFunctionCount) { + common::ByteStream out(1024, common::MOD_DEFAULT); + const uint8_t filter_byte = 1; + ASSERT_EQ(common::E_OK, common::SerializationUtil::write_var_uint(1, out)); + ASSERT_EQ(common::E_OK, out.write_buf(&filter_byte, 1)); + ASSERT_EQ(common::E_OK, + common::SerializationUtil::write_var_uint(256, out)); + ASSERT_EQ(common::E_OK, common::SerializationUtil::write_var_uint( + BloomFilter::MAX_HASH_FUNC_COUNT + 1, out)); + + BloomFilter filter; + EXPECT_EQ(common::E_TSFILE_CORRUPTED, filter.deserialize_from(out)); +} + +TEST(BloomfilterTest, RejectsFilterLengthBeyondRemainingInput) { + common::ByteStream out(1024, common::MOD_DEFAULT); + ASSERT_EQ(common::E_OK, + common::SerializationUtil::write_var_uint(1024, out)); + + BloomFilter filter; + EXPECT_EQ(common::E_TSFILE_CORRUPTED, filter.deserialize_from(out)); +} diff --git a/python/tests/test_tsfile_properties.py b/python/tests/test_tsfile_properties.py index 63621ce8a..de2d8c88f 100644 --- a/python/tests/test_tsfile_properties.py +++ b/python/tests/test_tsfile_properties.py @@ -25,6 +25,7 @@ FileWriteError, TableSchema, TSDataType, + TsFileCorruptedError, TsFileReader, TsFileTableWriter, TsFileWriter, @@ -77,3 +78,21 @@ def test_tsfile_table_writer_property_delegation(tmp_path): with TsFileReader(path) as reader: assert reader.get_tsfile_properties()["table-property"] == b"after\x00\xff" + + +def test_reader_reports_invalid_utf8_property_key(tmp_path): + path = tmp_path / "invalid-property-key.tsfile" + writer = TsFileWriter(os.fspath(path)) + writer.add_tsfile_property("invalid-key", b"value") + writer.close() + + file_bytes = path.read_bytes() + assert file_bytes.count(b"invalid-key") == 1 + path.write_bytes(file_bytes.replace(b"invalid-key", b"invalid-\xffey", 1)) + + with TsFileReader(os.fspath(path)) as reader: + with pytest.raises( + TsFileCorruptedError, + match="TsFile property key is not valid UTF-8", + ): + reader.get_tsfile_properties() diff --git a/python/tsfile/tsfile_reader.pyx b/python/tsfile/tsfile_reader.pyx index a9b891fa2..a2e8fe263 100644 --- a/python/tsfile/tsfile_reader.pyx +++ b/python/tsfile/tsfile_reader.pyx @@ -30,6 +30,7 @@ from libc.stdint cimport INT64_MIN, INT64_MAX, uint32_t, uintptr_t from tsfile.schema import TSDataType as TSDataTypePy from tsfile.schema import DeviceID, DeviceTimeseriesMetadataGroup +from tsfile.exceptions import TsFileCorruptedError from tsfile.tag_filter import ComparisonTagFilter, BetweenTagFilter, AndTagFilter, OrTagFilter, NotTagFilter from .date_utils import parse_int_to_date from .tsfile_cpp cimport * @@ -538,9 +539,14 @@ cdef class TsFileReaderPy: check_error(err_code) try: for i in range(property_count): - key = PyBytes_FromStringAndSize( - properties[i].key, properties[i].key_len - ).decode('utf-8') + try: + key = PyBytes_FromStringAndSize( + properties[i].key, properties[i].key_len + ).decode('utf-8') + except UnicodeDecodeError: + raise TsFileCorruptedError( + context="TsFile property key is not valid UTF-8" + ) from None if properties[i].is_null: result[key] = None else: From 6a07a62793cadb84b3d062c6b7846fb7424d5e1a Mon Sep 17 00:00:00 2001 From: ColinLee Date: Wed, 5 Aug 2026 18:29:19 +0800 Subject: [PATCH 3/4] Fix Windows TsFile binary file mode --- cpp/src/file/write_file.cc | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cpp/src/file/write_file.cc b/cpp/src/file/write_file.cc index 227520b71..68ac127ac 100644 --- a/cpp/src/file/write_file.cc +++ b/cpp/src/file/write_file.cc @@ -52,6 +52,12 @@ int WriteFile::create(const std::string& file_path, int flags, mode_t mode) { int WriteFile::do_create(int flags, mode_t mode) { int ret = E_OK; +#ifdef _WIN32 + // TsFile is a binary format. Callers of the C++ API may pass ordinary + // POSIX-style flags without O_BINARY; leaving the descriptor in text mode + // would translate byte 0x0A to 0x0D 0x0A and corrupt serialized metadata. + flags |= O_BINARY; +#endif // TODO make sure no same file exists fd_ = ::open(path_.c_str(), flags, mode); if (fd_ < 0) { From 7a266bdf6f3a6f756791c3fccd92ae18e4a5e042 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Thu, 6 Aug 2026 11:18:31 +0800 Subject: [PATCH 4/4] Preserve legacy empty BloomFilter metadata --- cpp/src/reader/bloom_filter.cc | 19 +++++++++++++++++++ cpp/test/common/tsfile_common_test.cc | 27 +++++++++++++++++++++++++++ cpp/test/reader/bloom_filter_test.cc | 12 ++++++++++++ 3 files changed, 58 insertions(+) diff --git a/cpp/src/reader/bloom_filter.cc b/cpp/src/reader/bloom_filter.cc index acba67f27..09e2c9a15 100644 --- a/cpp/src/reader/bloom_filter.cc +++ b/cpp/src/reader/bloom_filter.cc @@ -255,6 +255,25 @@ int BloomFilter::deserialize_from(ByteStream& in) { uint8_t* filter_data = nullptr; if (RET_FAIL(SerializationUtil::read_var_uint(filter_data_bytes_len, in))) { } else if (filter_data_bytes_len == 0) { + // Older C++ writers serialized an empty filter as three zero varints: + // byte length, bit count, and hash-function count. The Java-compatible + // encoding contains only the byte length. Probe the two legacy fields + // and restore the cursor when the following data is instead the + // TsFile property count from the current encoding. + const uint64_t legacy_fields_pos = in.read_pos(); + if (in.remaining_size() >= 2) { + uint32_t legacy_size = 0; + uint32_t legacy_hash_func_count = 0; + int probe_ret = SerializationUtil::read_var_uint(legacy_size, in); + if (probe_ret == E_OK && legacy_size == 0) { + probe_ret = SerializationUtil::read_var_uint( + legacy_hash_func_count, in); + } + if (probe_ret != E_OK || legacy_size != 0 || + legacy_hash_func_count != 0) { + in.set_read_pos(legacy_fields_pos); + } + } size_ = 0; hash_func_count_ = 0; return E_OK; diff --git a/cpp/test/common/tsfile_common_test.cc b/cpp/test/common/tsfile_common_test.cc index fcadaa440..5704afd19 100644 --- a/cpp/test/common/tsfile_common_test.cc +++ b/cpp/test/common/tsfile_common_test.cc @@ -482,6 +482,33 @@ TEST_F(TsFileMetaTest, SerializeDeserialize) { ASSERT_TRUE(new_meta.tsfile_properties_["null_key"].value.empty()); } +TEST_F(TsFileMetaTest, DeserializesLegacyEmptyBloomFilterEncoding) { + ASSERT_EQ(common::E_OK, + common::SerializationUtil::write_var_uint(0, *out_)); + ASSERT_EQ(common::E_OK, + common::SerializationUtil::write_var_uint(0, *out_)); + ASSERT_EQ(common::E_OK, common::SerializationUtil::write_i64(0, *out_)); + ASSERT_EQ(common::E_OK, + common::SerializationUtil::write_var_uint(0, *out_)); + ASSERT_EQ(common::E_OK, + common::SerializationUtil::write_var_uint(0, *out_)); + ASSERT_EQ(common::E_OK, + common::SerializationUtil::write_var_uint(0, *out_)); + ASSERT_EQ(common::E_OK, common::SerializationUtil::write_var_int(1, *out_)); + ASSERT_EQ(common::E_OK, + common::SerializationUtil::write_var_str("legacy", *out_)); + ASSERT_EQ(common::E_OK, common::SerializationUtil::write_var_int(3, *out_)); + ASSERT_EQ(common::E_OK, out_->write_buf("old", 3)); + + TsFileMeta meta(&pa_); + ASSERT_EQ(common::E_OK, meta.deserialize_from(*out_)); + ASSERT_EQ(1U, meta.tsfile_properties_.size()); + ASSERT_FALSE(meta.tsfile_properties_["legacy"].is_null); + EXPECT_EQ((std::vector{'o', 'l', 'd'}), + meta.tsfile_properties_["legacy"].value); + EXPECT_EQ(0U, out_->remaining_size()); +} + TEST_F(TsFileMetaTest, RejectsPropertyKeyLengthBeyondRemainingInput) { ASSERT_EQ(common::E_OK, common::SerializationUtil::write_var_uint(0, *out_)); diff --git a/cpp/test/reader/bloom_filter_test.cc b/cpp/test/reader/bloom_filter_test.cc index 71ceb643e..46a43e466 100644 --- a/cpp/test/reader/bloom_filter_test.cc +++ b/cpp/test/reader/bloom_filter_test.cc @@ -79,6 +79,18 @@ TEST(BloomfilterTest, EmptyFilterUsesJavaCompatibleEncoding) { EXPECT_EQ(0U, out.remaining_size()); } +TEST(BloomfilterTest, DeserializesLegacyEmptyFilterEncoding) { + common::ByteStream out(1024, common::MOD_DEFAULT); + ASSERT_EQ(common::E_OK, common::SerializationUtil::write_var_uint(0, out)); + ASSERT_EQ(common::E_OK, common::SerializationUtil::write_var_uint(0, out)); + ASSERT_EQ(common::E_OK, common::SerializationUtil::write_var_uint(0, out)); + + BloomFilter deserialized; + ASSERT_EQ(common::E_OK, deserialized.deserialize_from(out)); + EXPECT_TRUE(deserialized.is_empty()); + EXPECT_EQ(0U, out.remaining_size()); +} + TEST(BloomfilterTest, RejectsInvalidHashFunctionCount) { common::ByteStream out(1024, common::MOD_DEFAULT); const uint8_t filter_byte = 1;