PBS-42 feature: Implement basic support for COM_BINLOG_DUMP packet handling (part 2)

CMakeLists.txt

@@ -120,6 +120,8 @@ set(util_source_files
 
   src/util/ct_string.hpp
 
+  src/util/dynamic_byte_buffer_fwd.hpp
+
   src/util/exception_location_helpers.hpp
 
   src/util/file_operations_helpers.hpp

src/binsrv/basic_storage_backend.cpp

@@ -24,6 +24,7 @@
 
 #include "util/byte_range_fwd.hpp"
 #include "util/byte_span_fwd.hpp"
+#include "util/dynamic_byte_buffer_fwd.hpp"
 #include "util/exception_location_helpers.hpp"
 
 namespace binsrv {
@@ -33,7 +34,7 @@ basic_storage_backend::list_objects() {
   return do_list_objects();
 }
 
-[[nodiscard]] std::string
+[[nodiscard]] util::dynamic_byte_buffer
 basic_storage_backend::get_object(std::string_view name,
                                   const util::byte_range &range) {
   return do_get_object(name, range);

src/binsrv/basic_storage_backend.hpp

@@ -25,6 +25,7 @@
 #include "util/byte_range.hpp"
 #include "util/byte_span_fwd.hpp"
 #include "util/common_optional_types.hpp"
+#include "util/dynamic_byte_buffer_fwd.hpp"
 
 namespace binsrv {
 
@@ -42,7 +43,7 @@ class basic_storage_backend {
   virtual ~basic_storage_backend() = default;
 
   [[nodiscard]] storage_object_name_container list_objects();
-  [[nodiscard]] std::string
+  [[nodiscard]] util::dynamic_byte_buffer
   get_object(std::string_view name,
              const util::byte_range &range = util::byte_range{});
   // 'put_object' is an atomic overwrite: a concurrent / post-crash
@@ -78,7 +79,7 @@ class basic_storage_backend {
   bool stream_open_{false};
 
   [[nodiscard]] virtual storage_object_name_container do_list_objects() = 0;
-  [[nodiscard]] virtual std::string
+  [[nodiscard]] virtual util::dynamic_byte_buffer
   do_get_object(std::string_view name, const util::byte_range &range) = 0;
   virtual void do_put_object(std::string_view name,
                              util::const_byte_span content) = 0;

src/binsrv/filesystem_storage_backend.cpp

@@ -37,6 +37,7 @@
 
 #include "util/byte_range_fwd.hpp"
 #include "util/byte_span.hpp"
+#include "util/dynamic_byte_buffer_fwd.hpp"
 #include "util/exception_location_helpers.hpp"
 #include "util/file_operations_helpers.hpp"
 #include "util/native_file_operations_helpers.hpp"
@@ -115,7 +116,7 @@ filesystem_storage_backend::do_list_objects() {
   return result;
 }
 
-[[nodiscard]] std::string
+[[nodiscard]] util::dynamic_byte_buffer
 filesystem_storage_backend::do_get_object(std::string_view name,
                                           const util::byte_range &range) {
   const auto object_path{get_object_path(name)};
@@ -141,7 +142,7 @@ void filesystem_storage_backend::do_put_object(std::string_view name,
   tmp_object_path += tmp_storage_object_suffix;
 
   util::write_file_content("underlying tmp object file", tmp_object_path,
-                           util::as_string_view(content));
+                           content);
   // make the tmp file's content durable before the rename swaps it
   util::fsync(tmp_object_path);
 

src/binsrv/filesystem_storage_backend.hpp

@@ -44,7 +44,7 @@ class [[nodiscard]] filesystem_storage_backend final
 
   [[nodiscard]] storage_object_name_container do_list_objects() override;
 
-  [[nodiscard]] std::string
+  [[nodiscard]] util::dynamic_byte_buffer
   do_get_object(std::string_view name, const util::byte_range &range) override;
   void do_put_object(std::string_view name,
                      util::const_byte_span content) override;

src/binsrv/keyring_record_collection.cpp

@@ -26,6 +26,7 @@
 
 #include "binsrv/keyring_record.hpp"
 
+#include "util/byte_span.hpp"
 #include "util/exception_location_helpers.hpp"
 #include "util/file_operations_helpers.hpp"
 #include "util/nv_tuple_from_json.hpp"
@@ -37,7 +38,7 @@ keyring_record_collection::keyring_record_collection(std::string_view file_name)
   static constexpr std::size_t max_file_size{1048576U};
   const auto data = util::read_file_content("keyring record collection file",
                                             file_name, max_file_size);
-  auto json_value = boost::json::parse(data);
+  auto json_value = boost::json::parse(util::as_string_view(data));
   util::nv_tuple_from_json(json_value, impl_);
 
   validate();

src/binsrv/main_config.cpp

@@ -17,7 +17,6 @@
 
 #include <cstddef>
 #include <stdexcept>
-#include <string>
 #include <string_view>
 
 #include <boost/json/parse.hpp>
@@ -34,6 +33,7 @@
 // Needed for ssl_mode_type's operator <<
 #include "easymysql/ssl_mode_type.hpp" // IWYU pragma: keep
 
+#include "util/byte_span.hpp"
 #include "util/exception_location_helpers.hpp"
 #include "util/file_operations_helpers.hpp"
 #include "util/nv_tuple_from_json.hpp"
@@ -50,7 +50,7 @@ main_config::main_config(std::string_view file_name) {
         "configuration file is empty");
   }
 
-  auto json_value = boost::json::parse(file_content);
+  auto json_value = boost::json::parse(util::as_string_view(file_content));
   util::nv_tuple_from_json(json_value, impl_);
 
   validate();

src/binsrv/s3_storage_backend.cpp

@@ -68,6 +68,7 @@
 
 #include "util/byte_range.hpp"
 #include "util/byte_span.hpp"
+#include "util/dynamic_byte_buffer_fwd.hpp"
 #include "util/exception_location_helpers.hpp"
 
 namespace {
@@ -161,7 +162,7 @@ class s3_storage_backend::aws_context : private aws_context_base {
 
   [[nodiscard]] std::string get_bucket_region(const std::string &bucket) const;
 
-  [[nodiscard]] std::string get_object_into_string(
+  [[nodiscard]] util::dynamic_byte_buffer get_object_into_byte_buffer(
       const qualified_object_path &source,
       const util::byte_range &range = util::byte_range{}) const;
 
@@ -270,8 +271,8 @@ s3_storage_backend::aws_context::aws_context(
       GetNameForBucketLocationConstraint(model_region);
 }
 
-[[nodiscard]] std::string
-s3_storage_backend::aws_context::get_object_into_string(
+[[nodiscard]] util::dynamic_byte_buffer
+s3_storage_backend::aws_context::get_object_into_byte_buffer(
     const qualified_object_path &source, const util::byte_range &range) const {
   if (range.is_empty()) {
     return {};
@@ -283,7 +284,7 @@ s3_storage_backend::aws_context::get_object_into_string(
           "The requested S3 object range is too large to be loaded in memory");
     }
   }
-  std::string content;
+  util::dynamic_byte_buffer content;
   auto stream_handler{[&content, &range](std::size_t content_length,
                                          std::iostream &content_stream) {
     // TODO: check object length in advance before calling GetObject
@@ -305,10 +306,11 @@ s3_storage_backend::aws_context::get_object_into_string(
     }
 
     content.resize(content_length);
-    if (!content_stream.read(std::data(content),
+    // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast)
+    if (!content_stream.read(reinterpret_cast<char *>(std::data(content)),
                              static_cast<std::streamsize>(content_length))) {
       util::exception_location().raise<std::runtime_error>(
-          "cannot read S3 object content into a string");
+          "cannot read S3 object content into a byte buffer");
     }
     assert(content_stream.gcount() ==
            static_cast<std::streamsize>(content_length));
@@ -747,10 +749,10 @@ s3_storage_backend::do_list_objects() {
   return impl_->list_objects({.bucket = bucket_, .object_path = root_path_});
 }
 
-[[nodiscard]] std::string
+[[nodiscard]] util::dynamic_byte_buffer
 s3_storage_backend::do_get_object(std::string_view name,
                                   const util::byte_range &range) {
-  return impl_->get_object_into_string(
+  return impl_->get_object_into_byte_buffer(
       {.bucket = bucket_, .object_path = get_object_path(name)}, range);
 }
 

src/binsrv/s3_storage_backend.hpp

@@ -68,7 +68,7 @@ class [[nodiscard]] s3_storage_backend final : public basic_storage_backend {
 
   [[nodiscard]] storage_object_name_container do_list_objects() override;
 
-  [[nodiscard]] std::string
+  [[nodiscard]] util::dynamic_byte_buffer
   do_get_object(std::string_view name, const util::byte_range &range) override;
   void do_put_object(std::string_view name,
                      util::const_byte_span content) override;

src/binsrv/storage_core.cpp

@@ -579,9 +579,12 @@ storage_core::open_existing_binlog_file_internal(
 }
 
 void storage_core::load_binlog_index() {
-  const auto index_content{backend_->get_object(default_binlog_index_name)};
+  const auto index_content_buffer{
+      backend_->get_object(default_binlog_index_name)};
+  std::string index_content{util::as_string_view(index_content_buffer)};
+  // TODO: switch to std::ispanstream when clang has it implemented
   // opening in text mode
-  std::istringstream index_iss{index_content};
+  std::istringstream index_iss{std::move(index_content)};
   std::string current_line;
   while (std::getline(index_iss, current_line)) {
     if (current_line.empty()) {
@@ -657,7 +660,7 @@ void storage_core::save_binlog_index() const {
 
 void storage_core::load_metadata() {
   const auto metadata_content{backend_->get_object(metadata_name)};
-  const storage_metadata metadata{metadata_content};
+  const storage_metadata metadata{util::as_string_view(metadata_content)};
   replication_mode_ = metadata.root().get<"mode">();
   encryption_format_ = metadata.root().get<"encryption">();
 }
@@ -701,7 +704,7 @@ void storage_core::save_metadata() const {
     const events::composite_binlog_name &binlog_name) const {
   const auto content{
       backend_->get_object(generate_binlog_metadata_name(binlog_name))};
-  binlog_file_metadata metadata{content};
+  binlog_file_metadata metadata{util::as_string_view(content)};
 
   const auto &optional_encryption_metadata{metadata.root().get<"encryption">()};
   return binlog_record{

src/util/dynamic_byte_buffer_fwd.hpp

@@ -0,0 +1,28 @@
+// Copyright (c) 2023-2024 Percona and/or its affiliates.
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License, version 2.0,
+// as published by the Free Software Foundation.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License, version 2.0, for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program; if not, write to the Free Software
+// Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301  USA
+
+#ifndef UTIL_DYNAMIC_BYTE_BUFFER_FWD_HPP
+#define UTIL_DYNAMIC_BYTE_BUFFER_FWD_HPP
+
+#include <cstddef>
+#include <vector>
+
+namespace util {
+
+using dynamic_byte_buffer = std::vector<std::byte>;
+
+} // namespace util
+
+#endif // UTIL_DYNAMIC_BYTE_BUFFER_FWD_HPP

src/util/file_operations_helpers.cpp

@@ -26,11 +26,13 @@
 #include <string_view>
 
 #include "util/byte_range.hpp"
+#include "util/byte_span_fwd.hpp"
+#include "util/dynamic_byte_buffer_fwd.hpp"
 #include "util/exception_location_helpers.hpp"
 
 namespace util {
 
-[[nodiscard]] std::string
+[[nodiscard]] dynamic_byte_buffer
 read_file_content(std::string_view error_label,
                   const std::filesystem::path &path,
                   // NOLINTNEXTLINE(bugprone-easily-swappable-parameters)
@@ -83,8 +85,9 @@ read_file_content(std::string_view error_label,
         "cannot seek " + std::string{error_label} + " to the requested offset");
   }
 
-  std::string file_content(read_length, 'x');
-  if (!ifs.read(std::data(file_content),
+  dynamic_byte_buffer file_content(read_length);
+  // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast)
+  if (!ifs.read(reinterpret_cast<char *>(std::data(file_content)),
                 static_cast<std::streamoff>(read_length))) {
     exception_location().raise<std::runtime_error>(
         "cannot read " + std::string{error_label} + " content");
@@ -94,7 +97,7 @@ read_file_content(std::string_view error_label,
 
 void write_file_content(std::string_view error_label,
                         const std::filesystem::path &path,
-                        std::string_view content) {
+                        const_byte_span content) {
   // opening in binary mode with truncating
   std::ofstream ofs{};
   ofs.rdbuf()->pubsetbuf(nullptr, 0U);
@@ -105,7 +108,8 @@ void write_file_content(std::string_view error_label,
         "cannot open " + std::string{error_label} + " for writing");
   }
 
-  if (!ofs.write(std::data(content),
+  // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast)
+  if (!ofs.write(reinterpret_cast<const char *>(std::data(content)),
                  static_cast<std::streamoff>(std::size(content)))) {
     exception_location().raise<std::runtime_error>("cannot write data to " +
                                                    std::string{error_label});

src/util/file_operations_helpers.hpp

@@ -19,11 +19,12 @@
 #include <cstddef>
 #include <cstdint>
 #include <filesystem>
-#include <string>
 #include <string_view>
 
 #include "util/byte_range.hpp"
+#include "util/byte_span_fwd.hpp"
 #include "util/common_optional_types.hpp"
+#include "util/dynamic_byte_buffer_fwd.hpp"
 
 namespace util {
 
@@ -33,14 +34,14 @@ namespace util {
 // raises if actual length (either specified by the 'length' parameter
 // or determined by reading until the end of the file) is more than
 // 'max_length'
-[[nodiscard]] std::string
+[[nodiscard]] dynamic_byte_buffer
 read_file_content(std::string_view error_label,
                   const std::filesystem::path &path, std::size_t max_size,
                   const byte_range &range = byte_range{});
 
 void write_file_content(std::string_view error_label,
                         const std::filesystem::path &path,
-                        std::string_view content);
+                        const_byte_span content);
 
 } // namespace util