diff --git a/CMakeLists.txt b/CMakeLists.txt index a178660..42fdc8a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -120,6 +120,8 @@ set(util_source_files src/util/ct_string.hpp + src/util/dynamic_byte_buffer_fwd.hpp + src/util/exception_location_helpers.hpp src/util/file_operations_helpers.hpp diff --git a/src/binsrv/basic_storage_backend.cpp b/src/binsrv/basic_storage_backend.cpp index 81b385e..bd2ec9e 100644 --- a/src/binsrv/basic_storage_backend.cpp +++ b/src/binsrv/basic_storage_backend.cpp @@ -24,6 +24,7 @@ #include "util/byte_range_fwd.hpp" #include "util/byte_span_fwd.hpp" +#include "util/dynamic_byte_buffer_fwd.hpp" #include "util/exception_location_helpers.hpp" namespace binsrv { @@ -33,7 +34,7 @@ basic_storage_backend::list_objects() { return do_list_objects(); } -[[nodiscard]] std::string +[[nodiscard]] util::dynamic_byte_buffer basic_storage_backend::get_object(std::string_view name, const util::byte_range &range) { return do_get_object(name, range); diff --git a/src/binsrv/basic_storage_backend.hpp b/src/binsrv/basic_storage_backend.hpp index 01ce4e9..e40d66a 100644 --- a/src/binsrv/basic_storage_backend.hpp +++ b/src/binsrv/basic_storage_backend.hpp @@ -25,6 +25,7 @@ #include "util/byte_range.hpp" #include "util/byte_span_fwd.hpp" #include "util/common_optional_types.hpp" +#include "util/dynamic_byte_buffer_fwd.hpp" namespace binsrv { @@ -42,7 +43,7 @@ class basic_storage_backend { virtual ~basic_storage_backend() = default; [[nodiscard]] storage_object_name_container list_objects(); - [[nodiscard]] std::string + [[nodiscard]] util::dynamic_byte_buffer get_object(std::string_view name, const util::byte_range &range = util::byte_range{}); // 'put_object' is an atomic overwrite: a concurrent / post-crash @@ -78,7 +79,7 @@ class basic_storage_backend { bool stream_open_{false}; [[nodiscard]] virtual storage_object_name_container do_list_objects() = 0; - [[nodiscard]] virtual std::string + [[nodiscard]] virtual util::dynamic_byte_buffer do_get_object(std::string_view name, const util::byte_range &range) = 0; virtual void do_put_object(std::string_view name, util::const_byte_span content) = 0; diff --git a/src/binsrv/filesystem_storage_backend.cpp b/src/binsrv/filesystem_storage_backend.cpp index 3c9aefa..d6356c5 100644 --- a/src/binsrv/filesystem_storage_backend.cpp +++ b/src/binsrv/filesystem_storage_backend.cpp @@ -37,6 +37,7 @@ #include "util/byte_range_fwd.hpp" #include "util/byte_span.hpp" +#include "util/dynamic_byte_buffer_fwd.hpp" #include "util/exception_location_helpers.hpp" #include "util/file_operations_helpers.hpp" #include "util/native_file_operations_helpers.hpp" @@ -115,7 +116,7 @@ filesystem_storage_backend::do_list_objects() { return result; } -[[nodiscard]] std::string +[[nodiscard]] util::dynamic_byte_buffer filesystem_storage_backend::do_get_object(std::string_view name, const util::byte_range &range) { const auto object_path{get_object_path(name)}; @@ -141,7 +142,7 @@ void filesystem_storage_backend::do_put_object(std::string_view name, tmp_object_path += tmp_storage_object_suffix; util::write_file_content("underlying tmp object file", tmp_object_path, - util::as_string_view(content)); + content); // make the tmp file's content durable before the rename swaps it util::fsync(tmp_object_path); diff --git a/src/binsrv/filesystem_storage_backend.hpp b/src/binsrv/filesystem_storage_backend.hpp index 016f46c..7bfd463 100644 --- a/src/binsrv/filesystem_storage_backend.hpp +++ b/src/binsrv/filesystem_storage_backend.hpp @@ -44,7 +44,7 @@ class [[nodiscard]] filesystem_storage_backend final [[nodiscard]] storage_object_name_container do_list_objects() override; - [[nodiscard]] std::string + [[nodiscard]] util::dynamic_byte_buffer do_get_object(std::string_view name, const util::byte_range &range) override; void do_put_object(std::string_view name, util::const_byte_span content) override; diff --git a/src/binsrv/keyring_record_collection.cpp b/src/binsrv/keyring_record_collection.cpp index 2f5f2d7..3385590 100644 --- a/src/binsrv/keyring_record_collection.cpp +++ b/src/binsrv/keyring_record_collection.cpp @@ -26,6 +26,7 @@ #include "binsrv/keyring_record.hpp" +#include "util/byte_span.hpp" #include "util/exception_location_helpers.hpp" #include "util/file_operations_helpers.hpp" #include "util/nv_tuple_from_json.hpp" @@ -37,7 +38,7 @@ keyring_record_collection::keyring_record_collection(std::string_view file_name) static constexpr std::size_t max_file_size{1048576U}; const auto data = util::read_file_content("keyring record collection file", file_name, max_file_size); - auto json_value = boost::json::parse(data); + auto json_value = boost::json::parse(util::as_string_view(data)); util::nv_tuple_from_json(json_value, impl_); validate(); diff --git a/src/binsrv/main_config.cpp b/src/binsrv/main_config.cpp index f72a305..0c45c3e 100644 --- a/src/binsrv/main_config.cpp +++ b/src/binsrv/main_config.cpp @@ -17,7 +17,6 @@ #include #include -#include #include #include @@ -34,6 +33,7 @@ // Needed for ssl_mode_type's operator << #include "easymysql/ssl_mode_type.hpp" // IWYU pragma: keep +#include "util/byte_span.hpp" #include "util/exception_location_helpers.hpp" #include "util/file_operations_helpers.hpp" #include "util/nv_tuple_from_json.hpp" @@ -50,7 +50,7 @@ main_config::main_config(std::string_view file_name) { "configuration file is empty"); } - auto json_value = boost::json::parse(file_content); + auto json_value = boost::json::parse(util::as_string_view(file_content)); util::nv_tuple_from_json(json_value, impl_); validate(); diff --git a/src/binsrv/s3_storage_backend.cpp b/src/binsrv/s3_storage_backend.cpp index 7ebdc88..0086fad 100644 --- a/src/binsrv/s3_storage_backend.cpp +++ b/src/binsrv/s3_storage_backend.cpp @@ -68,6 +68,7 @@ #include "util/byte_range.hpp" #include "util/byte_span.hpp" +#include "util/dynamic_byte_buffer_fwd.hpp" #include "util/exception_location_helpers.hpp" namespace { @@ -161,7 +162,7 @@ class s3_storage_backend::aws_context : private aws_context_base { [[nodiscard]] std::string get_bucket_region(const std::string &bucket) const; - [[nodiscard]] std::string get_object_into_string( + [[nodiscard]] util::dynamic_byte_buffer get_object_into_byte_buffer( const qualified_object_path &source, const util::byte_range &range = util::byte_range{}) const; @@ -270,8 +271,8 @@ s3_storage_backend::aws_context::aws_context( GetNameForBucketLocationConstraint(model_region); } -[[nodiscard]] std::string -s3_storage_backend::aws_context::get_object_into_string( +[[nodiscard]] util::dynamic_byte_buffer +s3_storage_backend::aws_context::get_object_into_byte_buffer( const qualified_object_path &source, const util::byte_range &range) const { if (range.is_empty()) { return {}; @@ -283,7 +284,7 @@ s3_storage_backend::aws_context::get_object_into_string( "The requested S3 object range is too large to be loaded in memory"); } } - std::string content; + util::dynamic_byte_buffer content; auto stream_handler{[&content, &range](std::size_t content_length, std::iostream &content_stream) { // TODO: check object length in advance before calling GetObject @@ -305,10 +306,11 @@ s3_storage_backend::aws_context::get_object_into_string( } content.resize(content_length); - if (!content_stream.read(std::data(content), + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) + if (!content_stream.read(reinterpret_cast(std::data(content)), static_cast(content_length))) { util::exception_location().raise( - "cannot read S3 object content into a string"); + "cannot read S3 object content into a byte buffer"); } assert(content_stream.gcount() == static_cast(content_length)); @@ -747,10 +749,10 @@ s3_storage_backend::do_list_objects() { return impl_->list_objects({.bucket = bucket_, .object_path = root_path_}); } -[[nodiscard]] std::string +[[nodiscard]] util::dynamic_byte_buffer s3_storage_backend::do_get_object(std::string_view name, const util::byte_range &range) { - return impl_->get_object_into_string( + return impl_->get_object_into_byte_buffer( {.bucket = bucket_, .object_path = get_object_path(name)}, range); } diff --git a/src/binsrv/s3_storage_backend.hpp b/src/binsrv/s3_storage_backend.hpp index 1d4c7de..e012625 100644 --- a/src/binsrv/s3_storage_backend.hpp +++ b/src/binsrv/s3_storage_backend.hpp @@ -68,7 +68,7 @@ class [[nodiscard]] s3_storage_backend final : public basic_storage_backend { [[nodiscard]] storage_object_name_container do_list_objects() override; - [[nodiscard]] std::string + [[nodiscard]] util::dynamic_byte_buffer do_get_object(std::string_view name, const util::byte_range &range) override; void do_put_object(std::string_view name, util::const_byte_span content) override; diff --git a/src/binsrv/storage_core.cpp b/src/binsrv/storage_core.cpp index 57836bb..d9e47bc 100644 --- a/src/binsrv/storage_core.cpp +++ b/src/binsrv/storage_core.cpp @@ -579,9 +579,12 @@ storage_core::open_existing_binlog_file_internal( } void storage_core::load_binlog_index() { - const auto index_content{backend_->get_object(default_binlog_index_name)}; + const auto index_content_buffer{ + backend_->get_object(default_binlog_index_name)}; + std::string index_content{util::as_string_view(index_content_buffer)}; + // TODO: switch to std::ispanstream when clang has it implemented // opening in text mode - std::istringstream index_iss{index_content}; + std::istringstream index_iss{std::move(index_content)}; std::string current_line; while (std::getline(index_iss, current_line)) { if (current_line.empty()) { @@ -657,7 +660,7 @@ void storage_core::save_binlog_index() const { void storage_core::load_metadata() { const auto metadata_content{backend_->get_object(metadata_name)}; - const storage_metadata metadata{metadata_content}; + const storage_metadata metadata{util::as_string_view(metadata_content)}; replication_mode_ = metadata.root().get<"mode">(); encryption_format_ = metadata.root().get<"encryption">(); } @@ -701,7 +704,7 @@ void storage_core::save_metadata() const { const events::composite_binlog_name &binlog_name) const { const auto content{ backend_->get_object(generate_binlog_metadata_name(binlog_name))}; - binlog_file_metadata metadata{content}; + binlog_file_metadata metadata{util::as_string_view(content)}; const auto &optional_encryption_metadata{metadata.root().get<"encryption">()}; return binlog_record{ diff --git a/src/util/dynamic_byte_buffer_fwd.hpp b/src/util/dynamic_byte_buffer_fwd.hpp new file mode 100644 index 0000000..bf58998 --- /dev/null +++ b/src/util/dynamic_byte_buffer_fwd.hpp @@ -0,0 +1,28 @@ +// Copyright (c) 2023-2024 Percona and/or its affiliates. +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License, version 2.0, +// as published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License, version 2.0, for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +#ifndef UTIL_DYNAMIC_BYTE_BUFFER_FWD_HPP +#define UTIL_DYNAMIC_BYTE_BUFFER_FWD_HPP + +#include +#include + +namespace util { + +using dynamic_byte_buffer = std::vector; + +} // namespace util + +#endif // UTIL_DYNAMIC_BYTE_BUFFER_FWD_HPP diff --git a/src/util/file_operations_helpers.cpp b/src/util/file_operations_helpers.cpp index 667de99..597ca1e 100644 --- a/src/util/file_operations_helpers.cpp +++ b/src/util/file_operations_helpers.cpp @@ -26,11 +26,13 @@ #include #include "util/byte_range.hpp" +#include "util/byte_span_fwd.hpp" +#include "util/dynamic_byte_buffer_fwd.hpp" #include "util/exception_location_helpers.hpp" namespace util { -[[nodiscard]] std::string +[[nodiscard]] dynamic_byte_buffer read_file_content(std::string_view error_label, const std::filesystem::path &path, // NOLINTNEXTLINE(bugprone-easily-swappable-parameters) @@ -83,8 +85,9 @@ read_file_content(std::string_view error_label, "cannot seek " + std::string{error_label} + " to the requested offset"); } - std::string file_content(read_length, 'x'); - if (!ifs.read(std::data(file_content), + dynamic_byte_buffer file_content(read_length); + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) + if (!ifs.read(reinterpret_cast(std::data(file_content)), static_cast(read_length))) { exception_location().raise( "cannot read " + std::string{error_label} + " content"); @@ -94,7 +97,7 @@ read_file_content(std::string_view error_label, void write_file_content(std::string_view error_label, const std::filesystem::path &path, - std::string_view content) { + const_byte_span content) { // opening in binary mode with truncating std::ofstream ofs{}; ofs.rdbuf()->pubsetbuf(nullptr, 0U); @@ -105,7 +108,8 @@ void write_file_content(std::string_view error_label, "cannot open " + std::string{error_label} + " for writing"); } - if (!ofs.write(std::data(content), + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) + if (!ofs.write(reinterpret_cast(std::data(content)), static_cast(std::size(content)))) { exception_location().raise("cannot write data to " + std::string{error_label}); diff --git a/src/util/file_operations_helpers.hpp b/src/util/file_operations_helpers.hpp index 09cf02d..238309a 100644 --- a/src/util/file_operations_helpers.hpp +++ b/src/util/file_operations_helpers.hpp @@ -19,11 +19,12 @@ #include #include #include -#include #include #include "util/byte_range.hpp" +#include "util/byte_span_fwd.hpp" #include "util/common_optional_types.hpp" +#include "util/dynamic_byte_buffer_fwd.hpp" namespace util { @@ -33,14 +34,14 @@ namespace util { // raises if actual length (either specified by the 'length' parameter // or determined by reading until the end of the file) is more than // 'max_length' -[[nodiscard]] std::string +[[nodiscard]] dynamic_byte_buffer read_file_content(std::string_view error_label, const std::filesystem::path &path, std::size_t max_size, const byte_range &range = byte_range{}); void write_file_content(std::string_view error_label, const std::filesystem::path &path, - std::string_view content); + const_byte_span content); } // namespace util