From dabc74301e9ea69375ca7135f761c6629c1d4d00 Mon Sep 17 00:00:00 2001 From: Martin Mokros Date: Sat, 17 Oct 2026 19:26:23 +0100 Subject: [PATCH 01/23] Add Easy GUI (M1): guided raw recording -> lidar odometry app New raylib app apps/easy_gui for non-experts: pick a raw Mandeye folder, run lidar odometry with the precision forestry preset, get a session. Export (LAZ 1.4 + trajectory CSV) follows in M2. - Pre-flight check with plain-language findings: file pairing exactly as load_data does it (by position, unmatched trailing files reported), readable LAZ headers, single lidar id, calibration consistency (the cases where step 1 would abort() or exit()), IMU/lidar clock overlap, output folder location and a disk estimate (120 B per raw point of work space + 2.5x raw LAZ size of results). - Step 1 runs as a child process (lidar_odometry_step_1 ) for crash isolation and a real cancel (process group / job object kill). stdin is closed so load_data's cin.get() cannot hang; success is judged by session.mjs, since step 1 returns 0 after caught exceptions. - Step 1 runs on a staging folder of links to the recording, because load_data writes cache/ and preview/ next to its input: nothing is ever written into the recording (which may be the scanner's USB stick). - Parameters: preset 2 ("precise forestry") written with step 1's TomlIO, with threshould_output_filter lowered to filter_threshold_xy_inner. That filter is output-only: on a 117 M-point forest walk the trajectory is byte-identical to a manual preset 2 run, with 6.1 % more points kept. - Progress parsed from step 1's log (phases, per-chunk odometry progress, ETA) with a known-failure -> plain message table. - easy_gui --headless runs the same pipeline without a window. - easy_gui_tests (doctest): pre-flight on synthetic recordings, staging, preset file, child process, progress parser on a real log excerpt; plus integration tests against the real step 1 binary (cancel mid-odometry, broken recording) when EASY_GUI_TEST_RECORDING is set. No existing app, step 1 source or core_math file is changed; the root CMakeLists only adds the app and its tests. The Windows child-process code is written but not yet compiled on Windows. Co-Authored-By: Claude Opus 5.5 --- CMakeLists.txt | 2 + apps/easy_gui/CMakeLists.txt | 90 ++++ apps/easy_gui/src/child_process.cpp | 400 ++++++++++++++ apps/easy_gui/src/child_process.h | 58 ++ apps/easy_gui/src/easy_gui_main.cpp | 498 ++++++++++++++++++ apps/easy_gui/src/forestry_preset.cpp | 58 ++ apps/easy_gui/src/forestry_preset.h | 26 + apps/easy_gui/src/pipeline.cpp | 318 +++++++++++ apps/easy_gui/src/pipeline.h | 110 ++++ apps/easy_gui/src/preflight.cpp | 464 ++++++++++++++++ apps/easy_gui/src/preflight.h | 76 +++ apps/easy_gui/src/progress_parser.cpp | 209 ++++++++ apps/easy_gui/src/progress_parser.h | 63 +++ apps/easy_gui/src/settings.h | 24 + apps/easy_gui/src/staging.cpp | 69 +++ apps/easy_gui/src/staging.h | 20 + apps/easy_gui/tests/CMakeLists.txt | 26 + apps/easy_gui/tests/test_child_process.cpp | 76 +++ apps/easy_gui/tests/test_helpers.h | 97 ++++ apps/easy_gui/tests/test_main.cpp | 2 + .../tests/test_pipeline_integration.cpp | 100 ++++ apps/easy_gui/tests/test_preflight.cpp | 137 +++++ apps/easy_gui/tests/test_progress_parser.cpp | 157 ++++++ .../tests/test_staging_and_preset.cpp | 66 +++ 24 files changed, 3146 insertions(+) create mode 100644 apps/easy_gui/CMakeLists.txt create mode 100644 apps/easy_gui/src/child_process.cpp create mode 100644 apps/easy_gui/src/child_process.h create mode 100644 apps/easy_gui/src/easy_gui_main.cpp create mode 100644 apps/easy_gui/src/forestry_preset.cpp create mode 100644 apps/easy_gui/src/forestry_preset.h create mode 100644 apps/easy_gui/src/pipeline.cpp create mode 100644 apps/easy_gui/src/pipeline.h create mode 100644 apps/easy_gui/src/preflight.cpp create mode 100644 apps/easy_gui/src/preflight.h create mode 100644 apps/easy_gui/src/progress_parser.cpp create mode 100644 apps/easy_gui/src/progress_parser.h create mode 100644 apps/easy_gui/src/settings.h create mode 100644 apps/easy_gui/src/staging.cpp create mode 100644 apps/easy_gui/src/staging.h create mode 100644 apps/easy_gui/tests/CMakeLists.txt create mode 100644 apps/easy_gui/tests/test_child_process.cpp create mode 100644 apps/easy_gui/tests/test_helpers.h create mode 100644 apps/easy_gui/tests/test_main.cpp create mode 100644 apps/easy_gui/tests/test_pipeline_integration.cpp create mode 100644 apps/easy_gui/tests/test_preflight.cpp create mode 100644 apps/easy_gui/tests/test_progress_parser.cpp create mode 100644 apps/easy_gui/tests/test_staging_and_preset.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 8c0ce6b1..dc133cc3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -124,6 +124,7 @@ if(BUILD_TESTING) add_subdirectory(calib_core/tests) add_subdirectory(apps/lidar_odometry_step_1/tests) add_subdirectory(rosbags/tests) + add_subdirectory(apps/easy_gui/tests) endif() set(CORE_LIBRARIES core) @@ -147,6 +148,7 @@ add_subdirectory(apps/multi_session_registration) add_subdirectory(apps/multi_session_registration_legacy) add_subdirectory(apps/session_uncertainty_calculation) add_subdirectory(apps/session_to_session_rigid_registration) +add_subdirectory(apps/easy_gui) if(BUILD_WITH_UTILITY_APPLICATION) add_subdirectory(apps/split_multi_livox) diff --git a/apps/easy_gui/CMakeLists.txt b/apps/easy_gui/CMakeLists.txt new file mode 100644 index 00000000..5a6cec41 --- /dev/null +++ b/apps/easy_gui/CMakeLists.txt @@ -0,0 +1,90 @@ +cmake_minimum_required(VERSION 4.0.0) + +project(easy_gui) + +# easy_gui_core -- everything except the window: pre-flight check, staging, the +# step 1 child process and its progress parsing, and the pipeline that ties +# them together. A separate static library so the doctest suite in tests/ can +# link it without raylib. +# +# Lidar odometry itself is NOT compiled in: it runs as a child process +# (lidar_odometry_step_1 ), for crash isolation +# and a real cancel -- see src/child_process.h. Only two step 1 sources are +# compiled here, both by relative path like the other apps do: +# - lidar_odometry_utils.cpp for MLvxCalib (the pre-flight check mirrors +# load_data's calibration checks with step 1's own parser), +# - toml_io.cpp to write the parameter file with step 1's own TomlIO. +# WITH_GUI=0 is PUBLIC because LidarOdometryParams changes layout under +# WITH_GUI (clear_color), so every TU that sees it must agree. +add_library(easy_gui_core STATIC + src/child_process.cpp src/child_process.h + src/forestry_preset.cpp src/forestry_preset.h + src/pipeline.cpp src/pipeline.h + src/preflight.cpp src/preflight.h + src/progress_parser.cpp src/progress_parser.h + src/settings.h + src/staging.cpp src/staging.h + ../lidar_odometry_step_1/lidar_odometry_utils.cpp + ../lidar_odometry_step_1/toml_io.cpp +) + +target_compile_definitions(easy_gui_core + PUBLIC WITH_GUI=0 + PRIVATE HDMAPPING_BINARY_PREFIX="${HDMAPPING_BINARY_PREFIX}" UTL_PROFILER_DISABLE) + +target_include_directories(easy_gui_core + PUBLIC + src + ${REPOSITORY_DIRECTORY}/apps/lidar_odometry_step_1 + ${REPOSITORY_DIRECTORY}/core/include + ${THIRDPARTY_DIRECTORY} + ${EIGEN3_INCLUDE_DIR} + ${THIRDPARTY_DIRECTORY}/json/include + ${THIRDPARTY_DIRECTORY}/tomlplusplus/include + ${LASZIP_INCLUDE_DIR}/LASzip/include + ${THIRDPARTY_DIRECTORY}/observation_equations/codes + ${THIRDPARTY_DIRECTORY}/vqf/vqf/cpp + ${THIRDPARTY_DIRECTORY}/Fusion/Fusion) + +# core_math provides the Core/ndt.h + hash_utils symbols lidar_odometry_utils.cpp +# needs (the same reason step 1's own tests link core_no_gui); TBB and laszip for +# its load_point_cloud()/decimate(), which the pre-flight check never calls. +target_link_libraries(easy_gui_core PUBLIC + core_math + vqf + Fusion + unordered_dense::unordered_dense + spdlog::spdlog + UTL::include + ${PLATFORM_LASZIP_LIB} + ${PLATFORM_MISCELLANEOUS_LIBS}) + +if(MSVC) + target_compile_definitions(easy_gui_core PUBLIC _USE_MATH_DEFINES NOMINMAX) + target_compile_options(easy_gui_core PRIVATE /bigobj) +endif() + +# The window. raylib + rlImGui like step 2; core_pfd for the native folder picker. +add_executable(easy_gui src/easy_gui_main.cpp) + +target_link_libraries(easy_gui PRIVATE + easy_gui_core + core_pfd + raylib_widgets + raylib + imgui_raylib + rlimgui) + +# The step 1 program is part of Easy GUI at run time (it is started as a child +# process from the same directory), so building Easy GUI builds it too. +add_dependencies(easy_gui lidar_odometry_step_1) + +if(WIN32) + add_custom_command( + TARGET easy_gui + POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy $ $ + COMMAND_EXPAND_LISTS) +endif() + +hdmapping_install_app(easy_gui) diff --git a/apps/easy_gui/src/child_process.cpp b/apps/easy_gui/src/child_process.cpp new file mode 100644 index 00000000..c9c6fba3 --- /dev/null +++ b/apps/easy_gui/src/child_process.cpp @@ -0,0 +1,400 @@ +#include "child_process.h" + +#include +#include + +#ifdef _WIN32 +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#else +#include +#include +#include +#include +#include +extern char** environ; +#endif + +#ifdef __APPLE__ +#include +#endif + +namespace fs = std::filesystem; + +namespace easy_gui +{ +#ifdef _WIN32 + struct ChildProcess::Impl + { + HANDLE process = nullptr; + HANDLE job = nullptr; + HANDLE read_pipe = nullptr; + bool waited = false; + int exit_code = -1; + }; + + namespace + { + std::wstring widen(const std::string& s) + { + if (s.empty()) + return {}; + const int n = MultiByteToWideChar(CP_UTF8, 0, s.data(), static_cast(s.size()), nullptr, 0); + std::wstring out(n, L'\0'); + MultiByteToWideChar(CP_UTF8, 0, s.data(), static_cast(s.size()), out.data(), n); + return out; + } + + // Quoting rules of CommandLineToArgvW / the MSVC runtime: backslashes are literal except directly before a + // double quote, where they have to be doubled. + std::wstring quoteArgument(const std::wstring& arg) + { + if (!arg.empty() && arg.find_first_of(L" \t\n\v\"") == std::wstring::npos) + return arg; + std::wstring out = L"\""; + for (auto it = arg.begin();; ++it) + { + size_t backslashes = 0; + while (it != arg.end() && *it == L'\\') + { + ++it; + ++backslashes; + } + if (it == arg.end()) + { + out.append(backslashes * 2, L'\\'); + break; + } + if (*it == L'"') + out.append(backslashes * 2 + 1, L'\\'); + else + out.append(backslashes, L'\\'); + out.push_back(*it); + } + out.push_back(L'"'); + return out; + } + + std::string lastErrorMessage() + { + return "Windows error " + std::to_string(GetLastError()); + } + } // namespace + + ChildProcess::ChildProcess() + : impl_(new Impl) + { + } + + ChildProcess::~ChildProcess() + { + if (impl_->process && !impl_->waited) + { + kill(); + wait(); + } + if (impl_->read_pipe) + CloseHandle(impl_->read_pipe); + if (impl_->process) + CloseHandle(impl_->process); + if (impl_->job) + CloseHandle(impl_->job); + delete impl_; + } + + bool ChildProcess::start(const fs::path& executable, const std::vector& arguments, std::string& error) + { + SECURITY_ATTRIBUTES sa{ sizeof(SECURITY_ATTRIBUTES), nullptr, TRUE }; + HANDLE read_pipe = nullptr; + HANDLE write_pipe = nullptr; + if (!CreatePipe(&read_pipe, &write_pipe, &sa, 0)) + { + error = "Could not create a pipe (" + lastErrorMessage() + ")"; + return false; + } + SetHandleInformation(read_pipe, HANDLE_FLAG_INHERIT, 0); + + HANDLE nul = CreateFileW(L"NUL", GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE, &sa, OPEN_EXISTING, 0, nullptr); + + // The job object makes kill() take down the whole process tree and also kills the child if Easy GUI itself + // dies (JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE). + HANDLE job = CreateJobObjectW(nullptr, nullptr); + if (job) + { + JOBOBJECT_EXTENDED_LIMIT_INFORMATION info{}; + info.BasicLimitInformation.LimitFlags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE; + SetInformationJobObject(job, JobObjectExtendedLimitInformation, &info, sizeof(info)); + } + + std::wstring command_line = quoteArgument(executable.wstring()); + for (const auto& arg : arguments) + command_line += L" " + quoteArgument(widen(arg)); + + STARTUPINFOW si{}; + si.cb = sizeof(si); + si.dwFlags = STARTF_USESTDHANDLES; + si.hStdInput = nul; + si.hStdOutput = write_pipe; + si.hStdError = write_pipe; + + PROCESS_INFORMATION pi{}; + const BOOL ok = CreateProcessW( + executable.wstring().c_str(), + command_line.data(), + nullptr, + nullptr, + TRUE, + CREATE_NO_WINDOW | CREATE_SUSPENDED, + nullptr, + nullptr, + &si, + &pi); + const std::string create_error = ok ? std::string() : lastErrorMessage(); + + CloseHandle(write_pipe); + if (nul != INVALID_HANDLE_VALUE) + CloseHandle(nul); + + if (!ok) + { + CloseHandle(read_pipe); + if (job) + CloseHandle(job); + error = "Could not start '" + executable.string() + "' (" + create_error + ")"; + return false; + } + + if (job) + AssignProcessToJobObject(job, pi.hProcess); + ResumeThread(pi.hThread); + CloseHandle(pi.hThread); + + impl_->process = pi.hProcess; + impl_->job = job; + impl_->read_pipe = read_pipe; + return true; + } + + bool ChildProcess::readLine(std::string& line) + { + for (;;) + { + const auto newline = buffer_.find('\n'); + if (newline != std::string::npos) + { + line = buffer_.substr(0, newline); + buffer_.erase(0, newline + 1); + std::erase(line, '\r'); + return true; + } + if (eof_ || !impl_->read_pipe) + { + if (buffer_.empty()) + return false; + line.swap(buffer_); + buffer_.clear(); + std::erase(line, '\r'); + return true; + } + char chunk[4096]; + DWORD n = 0; + if (!ReadFile(impl_->read_pipe, chunk, sizeof(chunk), &n, nullptr) || n == 0) + eof_ = true; + else + buffer_.append(chunk, n); + } + } + + void ChildProcess::kill() + { + killed_ = true; + if (impl_->job) + TerminateJobObject(impl_->job, 1); + else if (impl_->process) + TerminateProcess(impl_->process, 1); + } + + int ChildProcess::wait() + { + if (!impl_->process) + return -1; + if (!impl_->waited) + { + WaitForSingleObject(impl_->process, INFINITE); + DWORD code = 0; + impl_->exit_code = GetExitCodeProcess(impl_->process, &code) ? static_cast(code) : -1; + impl_->waited = true; + } + return killed_ ? -1 : impl_->exit_code; + } + + fs::path currentExecutableDirectory() + { + std::wstring buffer(32768, L'\0'); + const DWORD n = GetModuleFileNameW(nullptr, buffer.data(), static_cast(buffer.size())); + if (n == 0 || n >= buffer.size()) + return {}; + buffer.resize(n); + return fs::path(buffer).parent_path(); + } + +#else // POSIX + + struct ChildProcess::Impl + { + pid_t pid = -1; + int read_fd = -1; + std::atomic reaped{ false }; + int exit_code = -1; + }; + + ChildProcess::ChildProcess() + : impl_(new Impl) + { + } + + ChildProcess::~ChildProcess() + { + if (impl_->pid > 0 && !impl_->reaped) + { + kill(); + wait(); + } + if (impl_->read_fd >= 0) + close(impl_->read_fd); + delete impl_; + } + + bool ChildProcess::start(const fs::path& executable, const std::vector& arguments, std::string& error) + { + int fds[2]; + if (pipe(fds) != 0) + { + error = std::string("Could not create a pipe (") + std::strerror(errno) + ")"; + return false; + } + // Only the child's copy of the write end may stay open, or the parent never sees end-of-output. + fcntl(fds[0], F_SETFD, FD_CLOEXEC); + fcntl(fds[1], F_SETFD, FD_CLOEXEC); + + posix_spawn_file_actions_t actions; + posix_spawn_file_actions_init(&actions); + posix_spawn_file_actions_addopen(&actions, STDIN_FILENO, "/dev/null", O_RDONLY, 0); + posix_spawn_file_actions_adddup2(&actions, fds[1], STDOUT_FILENO); + posix_spawn_file_actions_adddup2(&actions, fds[1], STDERR_FILENO); + + // Own process group, so kill() can signal everything the child starts. + posix_spawnattr_t attributes; + posix_spawnattr_init(&attributes); + posix_spawnattr_setflags(&attributes, POSIX_SPAWN_SETPGROUP); + posix_spawnattr_setpgroup(&attributes, 0); + + const std::string exe = executable.string(); + std::vector argv_storage; + argv_storage.reserve(arguments.size() + 1); + argv_storage.push_back(exe); + argv_storage.insert(argv_storage.end(), arguments.begin(), arguments.end()); + std::vector argv; + for (auto& a : argv_storage) + argv.push_back(a.data()); + argv.push_back(nullptr); + + pid_t pid = -1; + const int rc = posix_spawn(&pid, exe.c_str(), &actions, &attributes, argv.data(), environ); + + posix_spawn_file_actions_destroy(&actions); + posix_spawnattr_destroy(&attributes); + close(fds[1]); + + if (rc != 0) + { + close(fds[0]); + error = "Could not start '" + exe + "' (" + std::strerror(rc) + ")"; + return false; + } + + impl_->pid = pid; + impl_->read_fd = fds[0]; + return true; + } + + bool ChildProcess::readLine(std::string& line) + { + for (;;) + { + const auto newline = buffer_.find('\n'); + if (newline != std::string::npos) + { + line = buffer_.substr(0, newline); + buffer_.erase(0, newline + 1); + std::erase(line, '\r'); + return true; + } + if (eof_ || impl_->read_fd < 0) + { + if (buffer_.empty()) + return false; + line.swap(buffer_); + buffer_.clear(); + std::erase(line, '\r'); + return true; + } + char chunk[4096]; + const ssize_t n = read(impl_->read_fd, chunk, sizeof(chunk)); + if (n > 0) + buffer_.append(chunk, static_cast(n)); + else if (n < 0 && errno == EINTR) + continue; + else + eof_ = true; + } + } + + void ChildProcess::kill() + { + killed_ = true; + // SIGKILL rather than SIGTERM: step 1 installs no handlers, so there is nothing to shut down gracefully, and + // a stuck TBB worker must not be able to delay the cancel. + if (impl_->pid > 0 && !impl_->reaped) + ::kill(-impl_->pid, SIGKILL); + } + + int ChildProcess::wait() + { + if (impl_->pid <= 0) + return -1; + if (!impl_->reaped) + { + int status = 0; + pid_t r; + do + r = waitpid(impl_->pid, &status, 0); + while (r < 0 && errno == EINTR); + impl_->exit_code = (r == impl_->pid && WIFEXITED(status)) ? WEXITSTATUS(status) : -1; + impl_->reaped = true; + } + return impl_->exit_code; + } + + fs::path currentExecutableDirectory() + { +#ifdef __APPLE__ + uint32_t size = 0; + _NSGetExecutablePath(nullptr, &size); + std::string buffer(size, '\0'); + if (_NSGetExecutablePath(buffer.data(), &size) != 0) + return {}; + buffer.resize(std::strlen(buffer.c_str())); + std::error_code ec; + const fs::path resolved = fs::weakly_canonical(buffer, ec); + return (ec ? fs::path(buffer) : resolved).parent_path(); +#else + std::error_code ec; + const fs::path exe = fs::read_symlink("/proc/self/exe", ec); + return ec ? fs::path() : exe.parent_path(); +#endif + } +#endif +} // namespace easy_gui diff --git a/apps/easy_gui/src/child_process.h b/apps/easy_gui/src/child_process.h new file mode 100644 index 00000000..802f8c86 --- /dev/null +++ b/apps/easy_gui/src/child_process.h @@ -0,0 +1,58 @@ +#pragma once + +#include +#include +#include +#include +#include + +namespace easy_gui +{ + //! Runs one external program with stdout and stderr merged into a single pipe and stdin closed. + //! + //! Easy GUI runs lidar odometry as a child process instead of a thread because the step 1 sources have + //! abort()/exit() paths (corrupt LAZ, calibration mismatch, out of memory) that would take the whole GUI down, + //! and because compute_step_2 can only be paused, not cancelled. Killing the child is the cancel. + //! + //! stdin is closed on purpose: load_data waits on std::cin.get() before std::exit on a calibration mismatch, and + //! an EOF stdin lets it exit immediately instead of hanging forever. + //! + //! readLine() is meant for one reader thread; kill() may be called from any other thread and makes a blocked + //! readLine() return false once the child's end of the pipe closes. + class ChildProcess + { + public: + ChildProcess(); + ~ChildProcess(); // kills a still-running child and reaps it + ChildProcess(const ChildProcess&) = delete; + ChildProcess& operator=(const ChildProcess&) = delete; + + bool start(const std::filesystem::path& executable, const std::vector& arguments, std::string& error); + + //! Next output line without its line terminator ('\r' characters are removed too: step 1 prints progress + //! as "\rrunning iterations: ..."). Returns false at end of output. + bool readLine(std::string& line); + + //! Terminates the child (on POSIX its whole process group, so helper processes go too). Safe to call twice. + void kill(); + + //! Waits for the child to exit and returns its exit code, or -1 if it was killed by a signal / could not be + //! waited for. + int wait(); + + bool wasKilled() const + { + return killed_.load(); + } + + private: + struct Impl; + Impl* impl_ = nullptr; + std::string buffer_; + bool eof_ = false; + std::atomic killed_{ false }; + }; + + //! Directory of the running executable (empty if it cannot be determined). + std::filesystem::path currentExecutableDirectory(); +} // namespace easy_gui diff --git a/apps/easy_gui/src/easy_gui_main.cpp b/apps/easy_gui/src/easy_gui_main.cpp new file mode 100644 index 00000000..84f1073e --- /dev/null +++ b/apps/easy_gui/src/easy_gui_main.cpp @@ -0,0 +1,498 @@ +// HDMapping Easy GUI: raw Mandeye recording -> lidar odometry (precision forestry preset) -> point cloud + trajectory. +// See CLAUDE.md ("Easy GUI") for the decisions behind the pipeline. + +#include "pipeline.h" +#include "preflight.h" +#include "settings.h" + +#include +#include +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; +using namespace easy_gui; + +namespace +{ + const ImVec4 kGreen(0.35f, 0.80f, 0.40f, 1.0f); + const ImVec4 kAmber(0.95f, 0.70f, 0.20f, 1.0f); + const ImVec4 kRed(0.95f, 0.35f, 0.35f, 1.0f); + const ImVec4 kGrey(0.60f, 0.60f, 0.60f, 1.0f); + + fs::path homeFolder() + { +#ifdef _WIN32 + const char* home = std::getenv("USERPROFILE"); +#else + const char* home = std::getenv("HOME"); +#endif + return home ? fs::path(home) : fs::current_path(); + } + + //! ~/Documents/HDMapping Easy/, with _2, _3, ... appended if that already exists. Never next to + //! the recording: that may be the scanner's USB stick. + fs::path defaultOutputFolder(const fs::path& recording) + { + const fs::path base = homeFolder() / "Documents" / "HDMapping Easy"; + const std::string name = recording.filename().empty() ? recording.parent_path().filename().string() : recording.filename().string(); + std::error_code ec; + fs::path candidate = base / name; + for (int i = 2; fs::exists(candidate, ec); i++) + candidate = base / (name + "_" + std::to_string(i)); + return candidate; + } + + std::string formatDuration(double seconds) + { + const int s = static_cast(seconds + 0.5); + char buffer[32]; + if (s >= 3600) + std::snprintf(buffer, sizeof(buffer), "%dh %02dm", s / 3600, (s / 60) % 60); + else + std::snprintf(buffer, sizeof(buffer), "%dm %02ds", s / 60, s % 60); + return buffer; + } + + void wrappedColored(const ImVec4& color, const std::string& text) + { + ImGui::PushStyleColor(ImGuiCol_Text, color); + ImGui::TextWrapped("%s", text.c_str()); + ImGui::PopStyleColor(); + } + + struct App + { + fs::path recording; + fs::path output_folder; + bool output_chosen_by_user = false; + Settings settings; + + std::future preflight_future; + std::optional preflight; + fs::path preflight_for_recording; + fs::path preflight_for_output; + + Pipeline pipeline; + std::string start_error; + bool show_log = false; + + void setRecording(const fs::path& folder) + { + recording = folder; + if (!output_chosen_by_user) + output_folder = defaultOutputFolder(folder); + startPreflight(); + } + + void startPreflight() + { + if (recording.empty()) + return; + preflight.reset(); + preflight_for_recording = recording; + preflight_for_output = output_folder; + preflight_future = std::async(std::launch::async, runPreflight, recording, output_folder); + } + + bool checking() const + { + return preflight_future.valid(); + } + + void poll() + { + if (preflight_future.valid() && preflight_future.wait_for(std::chrono::seconds(0)) == std::future_status::ready) + preflight = preflight_future.get(); + } + + bool readyToStart() const + { + return preflight && preflight->ok() && preflight_for_recording == recording && preflight_for_output == output_folder && + !pipeline.busy(); + } + }; + + void drawInputs(App& app, bool locked) + { + ImGui::BeginDisabled(locked); + + ImGui::SeparatorText("1. Recording"); + ImGui::TextWrapped("The folder the Mandeye scanner wrote (lidar*.laz and imu*.csv files). You can also drag it onto this window."); + if (ImGui::Button("Choose recording...")) + { + const std::string folder = mandeye::fd::SelectFolder("Choose the Mandeye recording folder"); + if (!folder.empty()) + app.setRecording(folder); + } + ImGui::SameLine(); + ImGui::TextUnformatted(app.recording.empty() ? "(none chosen)" : app.recording.string().c_str()); + + ImGui::SeparatorText("2. Output folder"); + ImGui::TextWrapped( + "Results go here. It must be a new or empty folder outside the recording; nothing is ever written into the recording."); + if (ImGui::Button("Choose output folder...")) + { + const std::string folder = mandeye::fd::SelectFolder("Choose where to save the results"); + if (!folder.empty()) + { + app.output_folder = folder; + app.output_chosen_by_user = true; + app.startPreflight(); + } + } + ImGui::SameLine(); + ImGui::TextUnformatted( + app.output_folder.empty() ? "(set automatically once a recording is chosen)" : app.output_folder.string().c_str()); + + if (ImGui::CollapsingHeader("Advanced settings")) + { + ImGui::Indent(); + float near = static_cast(app.settings.near_min_range_m); + ImGui::SetNextItemWidth(220); + if (ImGui::SliderFloat( + "Closest points kept [m]", + &near, + static_cast(Settings::kNearMinRangeLowest), + static_cast(Settings::kNearMinRangeHighest), + "%.2f m")) + { + app.settings.near_min_range_m = near; + app.settings.clamp(); + } + ImGui::SameLine(); + ImGui::TextDisabled("(?)"); + if (ImGui::IsItemHovered()) + ImGui::SetTooltip( + "Points closer to the scanner than this are left out of the point cloud.\n" + "0.4 m (default) removes the operator's body and the handle.\n" + "Lower values keep more of the close surroundings but may include the operator."); + ImGui::Checkbox("Keep intermediate files (for troubleshooting)", &app.settings.keep_work_files); + ImGui::Unindent(); + } + + ImGui::EndDisabled(); + } + + void drawPreflight(App& app) + { + ImGui::SeparatorText("3. Check"); + if (app.recording.empty()) + { + ImGui::TextDisabled("Choose a recording first."); + return; + } + if (app.checking()) + { + ImGui::TextUnformatted("Checking the recording..."); + return; + } + if (!app.preflight) + { + if (ImGui::Button("Check recording")) + app.startPreflight(); + return; + } + if (app.preflight_for_recording != app.recording || app.preflight_for_output != app.output_folder) + { + if (ImGui::Button("Check again")) + app.startPreflight(); + return; + } + for (const auto& f : app.preflight->findings) + { + const char* tag = f.severity == Severity::Error ? "[X]" : f.severity == Severity::Warning ? "[!]" : "[OK]"; + const ImVec4& color = f.severity == Severity::Error ? kRed : f.severity == Severity::Warning ? kAmber : kGreen; + ImGui::TextColored(color, "%s", tag); + ImGui::SameLine(); + ImGui::TextWrapped("%s", f.message.c_str()); + } + if (!app.preflight->ok() && ImGui::Button("Check again")) + app.startPreflight(); + } + + void stageRow(const char* label, int index, int current, bool failed) + { + if (index < current) + ImGui::TextColored(kGreen, "[done]"); + else if (index == current) + ImGui::TextColored(failed ? kRed : kAmber, failed ? "[failed]" : "[ .. ]"); + else + ImGui::TextColored(kGrey, "[ ]"); + ImGui::SameLine(); + ImGui::TextUnformatted(label); + } + + void drawRun(App& app) + { + const PipelineStatus s = app.pipeline.status(); + ImGui::SeparatorText("4. Process"); + + if (s.state == PipelineState::Idle) + { + ImGui::BeginDisabled(!app.readyToStart()); + if (ImGui::Button("Start processing", ImVec2(220, 0))) + { + PipelineRequest request{ app.recording, app.output_folder, findStep1Executable(), app.settings }; + app.start_error.clear(); + app.pipeline.start(request, app.start_error); + } + ImGui::EndDisabled(); + if (!app.start_error.empty()) + wrappedColored(kRed, app.start_error); + return; + } + + // Stage list. Stage indices: 0 check+prepare, 1 odometry, 2 export (M2). + const bool failed = s.state == PipelineState::Failed; + int current = 0; + if (s.stage == PipelineStage::Odometry || s.stage == PipelineStage::Cleanup) + current = 1; + if (s.state == PipelineState::Succeeded) + current = 2; + stageRow("Check and prepare the recording", 0, current, failed && current == 0); + stageRow("Lidar odometry (precision forestry settings)", 1, current, failed && current == 1); + ImGui::TextColored(kGrey, "[ ]"); + ImGui::SameLine(); + ImGui::TextDisabled("Export point cloud and trajectory (coming in the next version)"); + + if (s.state == PipelineState::Running) + { + ImGui::Spacing(); + if (s.stage == PipelineStage::Odometry) + { + const double phase = s.step1.phaseFraction(); + std::string label = phaseLabel(s.step1.phase); + if (s.step1.phase == Step1Phase::Odometry && s.step1.chunks_total > 0) + label += " (" + std::to_string(s.step1.chunks_done) + " / " + std::to_string(s.step1.chunks_total) + ")"; + ImGui::TextUnformatted(label.c_str()); + if (phase >= 0.0) + ImGui::ProgressBar(static_cast(phase), ImVec2(-1, 0)); + else + ImGui::ProgressBar(-1.0f * static_cast(ImGui::GetTime()), ImVec2(-1, 0), "working..."); + + const double overall = s.step1.overallFraction(); + std::string eta = "Elapsed " + formatDuration(s.elapsed_s); + if (overall > 0.05) + eta += ", about " + formatDuration(s.elapsed_s * (1.0 - overall) / overall) + " left"; + ImGui::TextUnformatted("Overall"); + ImGui::ProgressBar(static_cast(overall), ImVec2(-1, 0), eta.c_str()); + } + else + { + ImGui::TextUnformatted(s.stage == PipelineStage::Cleanup ? "Cleaning up..." : "Preparing..."); + ImGui::ProgressBar(-1.0f * static_cast(ImGui::GetTime()), ImVec2(-1, 0), "working..."); + } + + ImGui::Spacing(); + if (ImGui::Button("Cancel", ImVec2(120, 0))) + ImGui::OpenPopup("Cancel processing?"); + if (ImGui::BeginPopupModal("Cancel processing?", nullptr, ImGuiWindowFlags_AlwaysAutoResize)) + { + ImGui::TextUnformatted("Stop processing? The work done so far will be discarded."); + if (ImGui::Button("Yes, stop", ImVec2(120, 0))) + { + app.pipeline.cancel(); + ImGui::CloseCurrentPopup(); + } + ImGui::SameLine(); + if (ImGui::Button("No, continue", ImVec2(120, 0))) + ImGui::CloseCurrentPopup(); + ImGui::EndPopup(); + } + return; + } + + ImGui::Spacing(); + if (s.state == PipelineState::Succeeded) + wrappedColored(kGreen, s.message + " (took " + formatDuration(s.elapsed_s) + ")"); + else if (s.state == PipelineState::Cancelled) + wrappedColored(kAmber, s.message); + else + { + wrappedColored(kRed, s.message); + if (!s.detail.empty()) + ImGui::TextDisabled("Technical detail: %s", s.detail.c_str()); + ImGui::TextDisabled("The full log is in %s", OutputLayout(app.output_folder).log_file.string().c_str()); + } + + if (ImGui::Button("Open output folder")) + OpenURL(app.output_folder.string().c_str()); + ImGui::SameLine(); + if (ImGui::Button("Process another recording")) + { + app.pipeline.reset(); + app.recording.clear(); + app.output_folder.clear(); + app.output_chosen_by_user = false; + app.preflight.reset(); + } + } + + void drawLog(App& app) + { + ImGui::Spacing(); + if (!ImGui::CollapsingHeader("Details (processing log)")) + return; + const auto lines = app.pipeline.logTail(); + ImGui::BeginChild("log", ImVec2(0, 240), ImGuiChildFlags_Borders, ImGuiWindowFlags_HorizontalScrollbar); + for (const auto& line : lines) + ImGui::TextUnformatted(line.c_str()); + if (ImGui::GetScrollY() >= ImGui::GetScrollMaxY() - 5.0f) // follow the end unless the user scrolled up + ImGui::SetScrollHereY(1.0f); + ImGui::EndChild(); + } +} // namespace + +namespace +{ + //! `easy_gui --headless `: the same pipeline without a window, for scripted runs + //! (the dev validation script) and remote machines. Exit code 0 = success, 1 = failed / refused, 2 = usage. + int runHeadless(const fs::path& recording, const fs::path& output_folder) + { + // Unbuffered, so progress shows up as it happens even when redirected to a file or pipe (MSVC treats _IOLBF as + // full buffering); the output is a line every few seconds. + std::setvbuf(stdout, nullptr, _IONBF, 0); + const PreflightResult check = runPreflight(recording, output_folder); + for (const auto& f : check.findings) + std::printf( + "%s %s\n", + f.severity == Severity::Error ? "[X]" + : f.severity == Severity::Warning ? "[!]" + : "[OK]", + f.message.c_str()); + if (!check.ok()) + return 1; + + Pipeline pipeline; + std::string error; + if (!pipeline.start(PipelineRequest{ recording, output_folder, findStep1Executable(), Settings{} }, error)) + { + std::printf("[X] %s\n", error.c_str()); + return 1; + } + std::string last; + while (pipeline.busy()) + { + const PipelineStatus s = pipeline.status(); + char line[256]; + const double f = s.step1.phaseFraction(); + std::snprintf( + line, + sizeof(line), + "%s: %s overall %.0f%%", + phaseLabel(s.step1.phase), + f >= 0.0 ? (std::to_string(static_cast(f * 100)) + "%,").c_str() : "", + s.step1.overallFraction() * 100.0); + if (last != line) + std::printf("[%s] %s\n", formatDuration(s.elapsed_s).c_str(), line); + last = line; + std::this_thread::sleep_for(std::chrono::seconds(5)); + } + const PipelineStatus s = pipeline.status(); + std::printf( + "%s %s (%s)\n", s.state == PipelineState::Succeeded ? "[OK]" : "[X]", s.message.c_str(), formatDuration(s.elapsed_s).c_str()); + if (!s.detail.empty()) + std::printf(" detail: %s\n", s.detail.c_str()); + return s.state == PipelineState::Succeeded ? 0 : 1; + } +} // namespace + +int main(int argc, char* argv[]) +{ + if (argc > 1 && std::string(argv[1]) == "--headless") + { + if (argc != 4) + { + std::printf("usage: %s --headless \n", argv[0]); + return 2; + } + return runHeadless(argv[2], argv[3]); + } + + // Dev aid: `--screenshot [recording]` renders ~2 s, saves the window and exits (layout checks without + // capturing the whole desktop). + std::string screenshot_file; + int first_positional = 1; + if (argc > 2 && std::string(argv[1]) == "--screenshot") + { + screenshot_file = argv[2]; + first_positional = 3; + } + + App app; + if (argc > first_positional && fs::is_directory(argv[first_positional])) + app.setRecording(argv[first_positional]); + + SetConfigFlags(FLAG_WINDOW_RESIZABLE | FLAG_MSAA_4X_HINT); + InitWindow(900, 780, ("HDMapping Easy GUI " + std::string(HDMAPPING_CONFIGURE_PROJECT_VERSION)).c_str()); + raylib_widgets::fitWindowToScreen(); + // raylib's default exit key (Esc) would close the window mid-run. + SetExitKey(KEY_NULL); + SetTargetFPS(30); + rlImGuiSetup(true); + + int frame = 0; + while (!WindowShouldClose()) + { + app.poll(); + const bool locked = app.pipeline.busy() || app.pipeline.status().state != PipelineState::Idle; + + if (IsFileDropped()) + { + FilePathList dropped = LoadDroppedFiles(); + if (dropped.count > 0 && !locked) + { + fs::path p(dropped.paths[0]); + std::error_code ec; + app.setRecording(fs::is_directory(p, ec) ? p : p.parent_path()); + } + UnloadDroppedFiles(dropped); + } + + BeginDrawing(); + ClearBackground(Color{ 30, 30, 30, 255 }); + rlImGuiBegin(); + + const ImGuiViewport* viewport = ImGui::GetMainViewport(); + ImGui::SetNextWindowPos(viewport->WorkPos); + ImGui::SetNextWindowSize(viewport->WorkSize); + ImGui::Begin( + "HDMapping Easy GUI", nullptr, ImGuiWindowFlags_NoDecoration | ImGuiWindowFlags_NoMove | ImGuiWindowFlags_NoSavedSettings); + + ImGui::TextUnformatted("HDMapping Easy GUI"); + ImGui::TextDisabled("Mandeye recording -> lidar odometry (precision forestry settings) -> point cloud and trajectory"); + ImGui::Spacing(); + + drawInputs(app, locked); + drawPreflight(app); + drawRun(app); + drawLog(app); + + ImGui::End(); + rlImGuiEnd(); + EndDrawing(); + + if (!screenshot_file.empty() && ++frame == 60) + { + TakeScreenshot(screenshot_file.c_str()); + break; + } + } + + // Closing the window while processing stops the child process (Pipeline's destructor cancels and joins). + app.pipeline.cancel(); + rlImGuiShutdown(); + CloseWindow(); + return 0; +} diff --git a/apps/easy_gui/src/forestry_preset.cpp b/apps/easy_gui/src/forestry_preset.cpp new file mode 100644 index 00000000..e2aaa57a --- /dev/null +++ b/apps/easy_gui/src/forestry_preset.cpp @@ -0,0 +1,58 @@ +#include "forestry_preset.h" + +#include + +#include + +namespace easy_gui +{ + void applyPrecisionForestryPreset(LidarOdometryParams& params) + { + params.decimation = 0.01; + params.in_out_params_indoor.resolution_X = 0.1; + params.in_out_params_indoor.resolution_Y = 0.1; + params.in_out_params_indoor.resolution_Z = 0.1; + + params.in_out_params_outdoor.resolution_X = 0.3; + params.in_out_params_outdoor.resolution_Y = 0.3; + params.in_out_params_outdoor.resolution_Z = 0.3; + + params.filter_threshold_xy_inner = 1.0; + params.filter_threshold_xy_outer = 70.0; + params.threshould_output_filter = 1.5; + + params.use_robust_and_accurate_lidar_odometry = false; + params.distance_bucket = 0.2; + params.polar_angle_deg = 10.0; + params.azimutal_angle_deg = 10.0; + params.robust_and_accurate_lidar_odometry_iterations = 20; + + params.max_distance_lidar = 70.0; + params.nr_iter = 500; + params.sliding_window_trajectory_length_threshold = 10000; + params.real_time_threshold_seconds = 10; + } + + void applyEasyGuiOutputSettings(LidarOdometryParams& params) + { + params.threshould_output_filter = params.filter_threshold_xy_inner; + } + + bool writeStep1ParameterFile(const std::filesystem::path& file, std::string& error) + { + // Heap + value-initialisation, as lidar_odometry_gui.cpp's --dump-default-params does: the NDT grid members + // carry no initialisers (and would be written to the TOML), and the struct is too large for comfort on the + // stack of the pipeline thread. + auto params = std::make_unique(); + applyPrecisionForestryPreset(*params); + applyEasyGuiOutputSettings(*params); + + TomlIO toml_io; + if (!toml_io.SaveParametersToTomlFile(file.string(), *params)) + { + error = "Could not write the processing settings file '" + file.string() + "'."; + return false; + } + return true; + } +} // namespace easy_gui diff --git a/apps/easy_gui/src/forestry_preset.h b/apps/easy_gui/src/forestry_preset.h new file mode 100644 index 00000000..1e107c74 --- /dev/null +++ b/apps/easy_gui/src/forestry_preset.h @@ -0,0 +1,26 @@ +#pragma once + +#include +#include + +struct LidarOdometryParams; + +namespace easy_gui +{ + //! "Presets -> 2 Velocity < 8km/h, larger spaces, precise forestry (generic)" of lidar_odometry_step_1. + //! + //! NOTE: the preset only exists as inline GUI code (lidar_odometry_gui.cpp, and a copy in + //! drag_folder_with_mandeye_data_and_drop_here-precision_forestry.cpp), so the values are repeated here. They + //! overwrite every field set_lidar_odometry_default_params() touches, so the result equals "start the step 1 GUI, + //! pick preset 2" regardless of which app's default function ran first. Keep in sync with lidar_odometry_gui.cpp. + void applyPrecisionForestryPreset(LidarOdometryParams& params); + + //! Easy GUI's only deviation from the preset: threshould_output_filter (output-only) is lowered to + //! filter_threshold_xy_inner, so step 1 exports every point the load filter kept, and the near-point export picks + //! up exactly the points the load filter removed. The trajectory is unaffected (see CLAUDE.md). + void applyEasyGuiOutputSettings(LidarOdometryParams& params); + + //! Writes the parameter file the step 1 child process runs with (value-initialised params + preset + Easy GUI + //! output settings), through step 1's own TomlIO so every serialisable field is present. + bool writeStep1ParameterFile(const std::filesystem::path& file, std::string& error); +} // namespace easy_gui diff --git a/apps/easy_gui/src/pipeline.cpp b/apps/easy_gui/src/pipeline.cpp new file mode 100644 index 00000000..77149901 --- /dev/null +++ b/apps/easy_gui/src/pipeline.cpp @@ -0,0 +1,318 @@ +#include "pipeline.h" + +#include "forestry_preset.h" +#include "staging.h" + +#include + +#include +#include +#include +#include + +namespace fs = std::filesystem; + +namespace easy_gui +{ + namespace + { + constexpr size_t kLogTailLines = 400; + + std::string timestamp() + { + const std::time_t now = std::time(nullptr); + std::tm tm{}; +#ifdef _WIN32 + localtime_s(&tm, &now); +#else + localtime_r(&now, &tm); +#endif + std::ostringstream out; + out << std::put_time(&tm, "%Y-%m-%d %H:%M:%S"); + return out.str(); + } + + const char* severityTag(Severity s) + { + return s == Severity::Error ? "ERROR" : s == Severity::Warning ? "WARNING" : "OK"; + } + } // namespace + + OutputLayout::OutputLayout(const fs::path& root_) + : root(root_) + , log_file(root_ / "easy_gui.log") + , work(root_ / "work") + , staging(root_ / "work" / "staging") + , step1_parameters(root_ / "work" / "easy_gui_step1.toml") + , session(root_ / "work" / "lio_session") + { + } + + fs::path findStep1Executable() + { + if (const char* overridden = std::getenv("HDMAPPING_STEP1_EXECUTABLE"); overridden && *overridden) + return fs::path(overridden); + + const fs::path dir = currentExecutableDirectory(); +#ifdef _WIN32 + const char* suffix = ".exe"; +#else + const char* suffix = ""; +#endif + for (const std::string name : + { std::string("lidar_odometry_step_1"), std::string(HDMAPPING_BINARY_PREFIX) + "lidar_odometry_step_1" }) + { + const fs::path candidate = dir / (name + suffix); + std::error_code ec; + if (fs::is_regular_file(candidate, ec)) + return candidate; + } + return {}; + } + + Pipeline::~Pipeline() + { + cancel(); + if (thread_.joinable()) + thread_.join(); + } + + bool Pipeline::start(const PipelineRequest& request, std::string& error) + { + if (busy_) + { + error = "Processing is already running."; + return false; + } + if (thread_.joinable()) + thread_.join(); + + request_ = request; + request_.settings.clamp(); + cancel_requested_ = false; + { + std::lock_guard lock(mutex_); + status_ = PipelineStatus{}; + status_.state = PipelineState::Running; + log_tail_.clear(); + started_ = std::chrono::steady_clock::now(); + } + busy_ = true; + thread_ = std::thread(&Pipeline::run, this); + return true; + } + + void Pipeline::cancel() + { + cancel_requested_ = true; + std::lock_guard lock(mutex_); + if (child_) + child_->kill(); + } + + void Pipeline::reset() + { + if (busy_) + return; + if (thread_.joinable()) + thread_.join(); + std::lock_guard lock(mutex_); + status_ = PipelineStatus{}; + log_tail_.clear(); + } + + PipelineStatus Pipeline::status() const + { + std::lock_guard lock(mutex_); + PipelineStatus s = status_; + if (s.state == PipelineState::Running) + s.elapsed_s = std::chrono::duration(std::chrono::steady_clock::now() - started_).count(); + return s; + } + + std::vector Pipeline::logTail() const + { + std::lock_guard lock(mutex_); + return { log_tail_.begin(), log_tail_.end() }; + } + + void Pipeline::log(const std::string& line) + { + if (log_file_.is_open()) + log_file_ << line << '\n'; + std::lock_guard lock(mutex_); + log_tail_.push_back(line); + while (log_tail_.size() > kLogTailLines) + log_tail_.pop_front(); + } + + void Pipeline::setStage(PipelineStage stage) + { + std::lock_guard lock(mutex_); + status_.stage = stage; + } + + void Pipeline::finish(PipelineState state, const std::string& message, const std::string& detail) + { + log("[easy_gui] " + timestamp() + " " + (state == PipelineState::Succeeded ? "finished: " : "stopped: ") + message); + if (!detail.empty()) + log("[easy_gui] detail: " + detail); + log_file_.close(); // finish() is the last thing a run logs + std::lock_guard lock(mutex_); + status_.state = state; + status_.message = message; + status_.detail = detail; + status_.elapsed_s = std::chrono::duration(std::chrono::steady_clock::now() - started_).count(); + } + + void Pipeline::run() + { + const OutputLayout out(request_.output_folder); + std::error_code ec; + + auto cleanupWork = [&](bool success) + { + setStage(PipelineStage::Cleanup); + if (request_.settings.keep_work_files) + return; + // The session stays on success: the export reads it, and an expert can open it in step 2. Everything + // else in work/ (links, step 1's cache/ and preview/) is disposable, and the cache can be gigabytes. + fs::remove_all(success ? out.staging : out.work, ec); + }; + + // --- 1. re-check: the recording or the disk may have changed since the user pressed "Check" + setStage(PipelineStage::Checking); + const PreflightResult check = runPreflight(request_.recording, request_.output_folder); + if (!check.ok()) + { + std::string first_error; + for (const auto& f : check.findings) + if (f.severity == Severity::Error) + { + first_error = f.message; + break; + } + finish(PipelineState::Failed, first_error); + busy_ = false; + return; + } + + if (!fs::create_directories(out.root, ec) && ec) + { + finish(PipelineState::Failed, "Could not create the output folder '" + out.root.string() + "' (" + ec.message() + ")."); + busy_ = false; + return; + } + log_file_.open(out.log_file, std::ios::out | std::ios::app); + + log("[easy_gui] HDMapping Easy GUI " + std::string(HDMAPPING_CONFIGURE_PROJECT_VERSION) + ", started " + timestamp()); + log("[easy_gui] recording: " + request_.recording.string()); + log("[easy_gui] output: " + out.root.string()); + log("[easy_gui] step 1 program: " + request_.step1_executable.string()); + { + std::ostringstream s; + s << "[easy_gui] settings: near_min_range_m=" << request_.settings.near_min_range_m + << " keep_work_files=" << request_.settings.keep_work_files; + log(s.str()); + } + for (const auto& f : check.findings) + log(std::string("[easy_gui] check ") + severityTag(f.severity) + ": " + f.message); + + // --- 2. staging folder + parameter file + setStage(PipelineStage::Staging); + std::string error; + std::string method; + if (!createStagingFolder(check.files, out.staging, method, error)) + { + cleanupWork(false); + finish(PipelineState::Failed, error); + busy_ = false; + return; + } + log("[easy_gui] staged " + std::to_string(check.files.pairs) + " file pairs as " + method); + if (!writeStep1ParameterFile(out.step1_parameters, error)) + { + cleanupWork(false); + finish(PipelineState::Failed, error); + busy_ = false; + return; + } + + // --- 3. lidar odometry in a child process + setStage(PipelineStage::Odometry); + if (request_.step1_executable.empty() || !fs::is_regular_file(request_.step1_executable, ec)) + { + cleanupWork(false); + finish( + PipelineState::Failed, + "Easy GUI could not find the lidar odometry program (lidar_odometry_step_1) next to itself. Please reinstall HDMapping.", + "looked for: " + request_.step1_executable.string()); + busy_ = false; + return; + } + + ChildProcess child; + const std::vector args = { out.staging.string(), out.step1_parameters.string(), out.session.string() }; + log("[easy_gui] running: " + request_.step1_executable.string() + " " + args[0] + " " + args[1] + " " + args[2]); + if (cancel_requested_ || !child.start(request_.step1_executable, args, error)) + { + cleanupWork(false); + if (cancel_requested_) + finish(PipelineState::Cancelled, "Processing was cancelled."); + else + finish(PipelineState::Failed, "The lidar odometry program could not be started.", error); + busy_ = false; + return; + } + { + std::lock_guard lock(mutex_); + child_ = &child; + } + if (cancel_requested_) // cancel() may have run between start() and publishing child_ + child.kill(); + + ProgressParser parser; + std::string line; + while (child.readLine(line)) + { + parser.feed(line); + log(line); + std::lock_guard lock(mutex_); + status_.step1 = parser.progress(); + } + const int exit_code = child.wait(); + { + std::lock_guard lock(mutex_); + child_ = nullptr; + status_.step1 = parser.progress(); + } + + const bool session_written = fs::is_regular_file(out.session / "session.mjs", ec); + if (cancel_requested_ || child.wasKilled()) + { + cleanupWork(false); + finish(PipelineState::Cancelled, "Processing was cancelled. Nothing was kept except the log file."); + } + // NOTE: step 1 exits with 0 after a caught exception, so the session file is the success criterion, not the + // exit code alone. + else if (exit_code != 0 || !session_written || parser.progress().failed) + { + std::string message = parser.progress().failure; + if (message.empty()) + message = exit_code != 0 ? "Lidar odometry stopped unexpectedly (exit code " + std::to_string(exit_code) + ")." + : "Lidar odometry finished without writing a result."; + cleanupWork(false); + finish(PipelineState::Failed, message, parser.progress().last_error_line); + } + else + { + { + std::lock_guard lock(mutex_); + status_.session_folder = out.session; + } + cleanupWork(true); + finish(PipelineState::Succeeded, "Lidar odometry finished. The session is in '" + out.session.string() + "'."); + } + busy_ = false; + } +} // namespace easy_gui diff --git a/apps/easy_gui/src/pipeline.h b/apps/easy_gui/src/pipeline.h new file mode 100644 index 00000000..e3368350 --- /dev/null +++ b/apps/easy_gui/src/pipeline.h @@ -0,0 +1,110 @@ +#pragma once + +#include "child_process.h" +#include "preflight.h" +#include "progress_parser.h" +#include "settings.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace easy_gui +{ + struct PipelineRequest + { + std::filesystem::path recording; + std::filesystem::path output_folder; + std::filesystem::path step1_executable; + Settings settings; + }; + + enum class PipelineState + { + Idle, + Running, + Succeeded, + Failed, + Cancelled + }; + + enum class PipelineStage + { + Checking, + Staging, + Odometry, + Cleanup + }; + + struct PipelineStatus + { + PipelineState state = PipelineState::Idle; + PipelineStage stage = PipelineStage::Checking; + Step1Progress step1; + std::string message; // plain-language outcome / failure reason + std::string detail; // technical detail for the "details" view + double elapsed_s = 0.0; + std::filesystem::path session_folder; // step 1 session (session.mjs) once odometry succeeded + }; + + //! Layout of an Easy GUI output folder. + struct OutputLayout + { + explicit OutputLayout(const std::filesystem::path& root); + std::filesystem::path root; + std::filesystem::path log_file; // easy_gui.log: everything, including the full step 1 output + std::filesystem::path work; // work/: intermediate data + std::filesystem::path staging; // work/staging: links to the recording; step 1 puts cache/ and preview/ here + std::filesystem::path step1_parameters; // work/easy_gui_step1.toml + std::filesystem::path session; // work/lio_session: step 1 output (session.mjs, scan_lio_N.laz, ...) + }; + + //! Locates lidar_odometry_step_1 next to the Easy GUI executable (plain build name or the installed, + //! HDMAPPING_BINARY_PREFIX-ed one). The HDMAPPING_STEP1_EXECUTABLE environment variable overrides the search. + std::filesystem::path findStep1Executable(); + + //! Runs the whole Easy GUI processing on a worker thread: re-check, stage, step 1 in a child process, clean up. + //! The GUI polls status() every frame; cancel() kills the child process. + class Pipeline + { + public: + ~Pipeline(); + + bool start(const PipelineRequest& request, std::string& error); + void cancel(); + //! Back to Idle after a finished run (no-op while busy). + void reset(); + bool busy() const + { + return busy_.load(); + } + + PipelineStatus status() const; + //! Last lines of the log, oldest first. + std::vector logTail() const; + + private: + void run(); + void log(const std::string& line); + void setStage(PipelineStage stage); + void finish(PipelineState state, const std::string& message, const std::string& detail = ""); + + PipelineRequest request_; + std::thread thread_; + std::atomic busy_{ false }; + std::atomic cancel_requested_{ false }; + + mutable std::mutex mutex_; + PipelineStatus status_; + std::deque log_tail_; + std::chrono::steady_clock::time_point started_; + ChildProcess* child_ = nullptr; // valid while the odometry child runs; guarded by mutex_ + std::ofstream log_file_; // only touched by the worker thread + }; +} // namespace easy_gui diff --git a/apps/easy_gui/src/preflight.cpp b/apps/easy_gui/src/preflight.cpp new file mode 100644 index 00000000..a47d923d --- /dev/null +++ b/apps/easy_gui/src/preflight.cpp @@ -0,0 +1,464 @@ +#include "preflight.h" + +#include + +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; + +namespace easy_gui +{ + namespace + { + std::string lower(std::string s) + { + std::transform( + s.begin(), + s.end(), + s.begin(), + [](unsigned char c) + { + return static_cast(std::tolower(c)); + }); + return s; + } + + std::string formatBytes(uint64_t bytes) + { + std::ostringstream out; + out.precision(1); + out << std::fixed << static_cast(bytes) / 1e9 << " GB"; + return out.str(); + } + + std::string names(const std::vector& list, size_t max_shown = 5) + { + std::string out; + for (size_t i = 0; i < list.size() && i < max_shown; i++) + out += (i ? ", " : "") + list[i]; + if (list.size() > max_shown) + out += " and " + std::to_string(list.size() - max_shown) + " more"; + return out; + } + + // Last 4 characters of the file stem, as load_data's get4index. + std::string index4(const fs::path& p) + { + const auto s = p.stem().string(); + return s.size() > 4 ? s.substr(s.size() - 4) : s; + } + + struct LazSummary + { + bool readable = false; + uint64_t points = 0; + double first_time = std::numeric_limits::max(); + double last_time = std::numeric_limits::lowest(); + std::set lidar_ids; + }; + + // Header + a sample of points from the start and the end of the file: enough to check the lidar id and the + // time range without decompressing whole files. + LazSummary summarizeLaz(const fs::path& file) + { + constexpr uint64_t kSample = 20000; + LazSummary s; + laszip_POINTER reader = nullptr; + if (laszip_create(&reader)) + return s; + + laszip_BOOL compressed = 0; + laszip_header* header = nullptr; + laszip_point* point = nullptr; + if (laszip_open_reader(reader, file.string().c_str(), &compressed) || laszip_get_header_pointer(reader, &header) || + laszip_get_point_pointer(reader, &point)) + { + laszip_destroy(reader); + return s; + } + + s.points = header->number_of_point_records ? header->number_of_point_records : header->extended_number_of_point_records; + s.readable = true; + + auto sample = [&](uint64_t begin, uint64_t end) + { + for (uint64_t i = begin; i < end; i++) + { + if (laszip_read_point(reader)) + { + s.readable = false; + return; + } + s.lidar_ids.insert(point->user_data); + if (point->gps_time != 0.0) + { + s.first_time = std::min(s.first_time, point->gps_time); + s.last_time = std::max(s.last_time, point->gps_time); + } + } + }; + + sample(0, std::min(kSample, s.points)); + if (s.readable && s.points > kSample) + { + const uint64_t tail_begin = std::max(kSample, s.points - std::min(1000, s.points)); + if (laszip_seek_point(reader, static_cast(tail_begin))) + s.readable = false; + else + sample(tail_begin, s.points); + } + + laszip_close_reader(reader); + laszip_destroy(reader); + return s; + } + + // IMU timestamps (ns in the file) of the first and last sample, in seconds. Handles the named-column format + // and the legacy headerless one, like load_imu(). + bool imuTimeRange(const fs::path& first_file, const fs::path& last_file, double& t_first, double& t_last) + { + auto split = [](const std::string& line) + { + std::vector out; + std::string token; + for (char c : line) + { + if (c == ' ' || c == ',' || c == '\t' || c == '\r') + { + if (!token.empty()) + out.push_back(token); + token.clear(); + } + else + token += c; + } + if (!token.empty()) + out.push_back(token); + return out; + }; + + auto timestampColumn = [&](const fs::path& file, size_t& column, bool& has_header) -> bool + { + std::ifstream in(file); + std::string line; + if (!std::getline(in, line)) + return false; + const auto cols = split(line); + has_header = false; + column = 0; + for (size_t i = 0; i < cols.size(); i++) + if (cols[i] == "timestamp") + { + has_header = true; + column = i; + } + return true; + }; + + auto parseTime = [&](const std::string& line, size_t column, double& t) -> bool + { + const auto cols = split(line); + if (column >= cols.size()) + return false; + try + { + t = std::stod(cols[column]) / 1e9; + } catch (...) + { + return false; + } + return t > 0.0; + }; + + size_t column = 0; + bool has_header = false; + if (!timestampColumn(first_file, column, has_header)) + return false; + + { + std::ifstream in(first_file); + std::string line; + if (has_header) + std::getline(in, line); + bool found = false; + while (!found && std::getline(in, line)) + found = parseTime(line, column, t_first); + if (!found) + return false; + } + { + size_t last_column = 0; + bool last_has_header = false; + if (!timestampColumn(last_file, last_column, last_has_header)) + return false; + std::ifstream in(last_file); + std::string line; + std::string last_good; + double t = 0.0; + bool first = true; + while (std::getline(in, line)) + { + if (first && last_has_header) + { + first = false; + continue; + } + first = false; + if (parseTime(line, last_column, t)) + t_last = t; + } + } + return t_last > t_first; + } + } // namespace + + bool isSameOrInside(const fs::path& inner, const fs::path& outer) + { + std::error_code ec; + const fs::path a = fs::weakly_canonical(fs::absolute(inner, ec), ec).lexically_normal(); + const fs::path b = fs::weakly_canonical(fs::absolute(outer, ec), ec).lexically_normal(); + auto ai = a.begin(); + for (auto bi = b.begin(); bi != b.end(); ++bi, ++ai) + { + if (bi->empty()) // trailing separator + continue; + if (ai == a.end() || *ai != *bi) + return false; + } + return true; + } + + uint64_t estimateWorkBytes(uint64_t total_points) + { + return total_points * 120; + } + + uint64_t estimateResultBytes(uint64_t raw_laz_bytes) + { + // session ~0.85x + export ~1.5x + CSV, rounded up to 2.5x the raw LAZ size. + return raw_laz_bytes * 5 / 2; + } + + RecordingFiles classifyRecordingFiles(const fs::path& recording) + { + RecordingFiles files; + std::vector entries; + std::error_code ec; + for (const auto& entry : fs::directory_iterator(recording, ec)) + if (!entry.is_directory(ec)) + entries.push_back(entry.path()); + std::sort(entries.begin(), entries.end()); + + for (const auto& p : entries) + { + const std::string ext = lower(p.extension().string()); + if (ext == ".csv") + files.csv_files.push_back(p); + else if (ext == ".laz" || ext == ".las") + files.laz_files.push_back(p); + else if (ext == ".json") + { + const std::string stem = p.stem().string(); + if (!stem.starts_with("status") && !stem.starts_with("cam0") && !stem.starts_with("cam1") && files.calibration_file.empty()) + files.calibration_file = p; + } + else if (ext == ".mjc" && files.calibration_file.empty()) + files.calibration_file = p; + else if (ext == ".sn" && files.sn_file.empty()) + files.sn_file = p; + } + files.pairs = std::min(files.laz_files.size(), files.csv_files.size()); + return files; + } + + PreflightResult runPreflight(const fs::path& recording, const fs::path& output_folder) + { + PreflightResult r; + auto add = [&r](Severity s, std::string m) + { + r.findings.push_back({ s, std::move(m) }); + }; + + std::error_code ec; + if (recording.empty() || !fs::is_directory(recording, ec)) + { + add(Severity::Error, "The recording folder '" + recording.string() + "' does not exist or is not a folder."); + return r; + } + + r.files = classifyRecordingFiles(recording); + const auto& f = r.files; + + if (f.laz_files.empty() || f.csv_files.empty()) + { + add(Severity::Error, + std::string("This folder does not look like a Mandeye recording: ") + + (f.laz_files.empty() ? "there are no point cloud files (lidar*.laz)" : "there are no IMU files (imu*.csv)") + + ". Choose the folder the scanner wrote, for example 'continousScanning_0012'."); + return r; + } + + // --- pairing (by position, as load_data does) + if (f.laz_files.size() != f.csv_files.size()) + { + const auto& longer = f.laz_files.size() > f.csv_files.size() ? f.laz_files : f.csv_files; + std::vector skipped; + for (size_t i = f.pairs; i < longer.size(); i++) + skipped.push_back(longer[i].filename().string()); + add(Severity::Warning, + "There are " + std::to_string(f.laz_files.size()) + " point cloud files but " + std::to_string(f.csv_files.size()) + + " IMU files. The unmatched file(s) at the end will not be processed: " + names(skipped) + + ". This is normal when the scanner stopped between two files."); + } + std::vector misaligned; + for (size_t i = 0; i < f.pairs; i++) + if (index4(f.laz_files[i]) != index4(f.csv_files[i])) + misaligned.push_back(f.laz_files[i].filename().string() + " / " + f.csv_files[i].filename().string()); + if (!misaligned.empty()) + add(Severity::Warning, + "Some file numbers do not match (" + names(misaligned, 3) + + "). Files are used in order, so a missing file in the middle of the recording can leave a gap in the result."); + + // --- point cloud files: readable, one lidar, time range + std::vector unreadable; + std::set lidar_ids; + double lidar_first = std::numeric_limits::max(); + double lidar_last = std::numeric_limits::lowest(); + for (size_t i = 0; i < f.pairs; i++) + { + const auto s = summarizeLaz(f.laz_files[i]); + if (!s.readable) + { + unreadable.push_back(f.laz_files[i].filename().string()); + continue; + } + r.total_points += s.points; + r.raw_laz_bytes += fs::file_size(f.laz_files[i], ec); + lidar_ids.insert(s.lidar_ids.begin(), s.lidar_ids.end()); + lidar_first = std::min(lidar_first, s.first_time); + lidar_last = std::max(lidar_last, s.last_time); + } + if (!unreadable.empty()) + add(Severity::Error, + "These point cloud files cannot be read (damaged, or not fully copied from the scanner): " + names(unreadable) + + ". Copy the recording again, or remove the damaged files at the end of the recording."); + if (lidar_ids.size() > 1) + { + std::string ids; + for (int id : lidar_ids) + ids += (ids.empty() ? "" : ", ") + std::to_string(id); + add(Severity::Error, + "This recording contains data from more than one lidar (ids " + ids + + "). Easy GUI supports single-lidar recordings only; use the standard HDMapping apps for this one."); + } + else if (lidar_ids.size() == 1) + r.lidar_id = *lidar_ids.begin(); + + // --- calibration (mirrors load_data's checks, which otherwise end in std::exit) + const auto id_to_sn = MLvxCalib::GetIdToSnMapping(f.sn_file.string()); + if (!f.calibration_file.empty()) + { + const auto calibration = MLvxCalib::GetCalibrationFromFile(f.calibration_file.string()); + const std::string imu_sn = MLvxCalib::GetImuSnToUse(f.calibration_file.string()); + if (calibration.empty()) + add(Severity::Warning, + "The calibration file '" + f.calibration_file.filename().string() + "' could not be read and will be ignored."); + else if (calibration.size() > 1) + add(Severity::Error, + "The calibration file '" + f.calibration_file.filename().string() + + "' describes more than one lidar. Easy GUI supports single-lidar recordings only."); + else + { + for (const auto& [id, sn] : id_to_sn) + if (!calibration.contains(sn)) + add(Severity::Error, + "The calibration file '" + f.calibration_file.filename().string() + + "' belongs to a different scanner (this recording was made with lidar " + sn + ")."); + if (!calibration.contains(imu_sn)) + add(Severity::Error, + "The calibration file '" + f.calibration_file.filename().string() + "' does not name a valid IMU ('" + imu_sn + + "' is not one of its sensors)."); + } + } + if (id_to_sn.size() > 1) + add(Severity::Error, + "The scanner serial-number file lists more than one lidar. Easy GUI supports single-lidar recordings only."); + + // --- IMU vs lidar clocks + double imu_first = 0.0, imu_last = 0.0; + if (!imuTimeRange(f.csv_files.front(), f.csv_files[f.pairs - 1], imu_first, imu_last)) + add(Severity::Error, + "The IMU files could not be read (no valid timestamps in '" + f.csv_files.front().filename().string() + "')."); + else + { + r.duration_s = imu_last - imu_first; + if (lidar_last > lidar_first) + { + const double overlap = std::min(imu_last, lidar_last) - std::max(imu_first, lidar_first); + if (overlap <= 0.0) + add(Severity::Error, + "The IMU and point cloud timestamps do not overlap, so they cannot be combined. The recording is probably " + "damaged."); + } + if (r.duration_s < 20.0) + add(Severity::Warning, + "The recording is very short (" + std::to_string(static_cast(r.duration_s)) + + " s). Lidar odometry needs some movement; results may be poor."); + } + + // --- output folder + r.estimated_work_bytes = estimateWorkBytes(r.total_points); + r.estimated_result_bytes = estimateResultBytes(r.raw_laz_bytes); + r.estimated_disk_bytes = r.estimated_work_bytes + r.estimated_result_bytes; + const std::string need = "about " + formatBytes(r.estimated_disk_bytes) + " (" + formatBytes(r.estimated_work_bytes) + + " temporary during processing + " + formatBytes(r.estimated_result_bytes) + " of results)"; + std::string disk_summary; + if (output_folder.empty()) + add(Severity::Error, "Choose an output folder."); + else if (isSameOrInside(output_folder, recording)) + add(Severity::Error, + "The output folder is inside the recording folder. Choose a folder elsewhere: Easy GUI never writes into the recording " + "(it may be on the scanner's USB stick)."); + else + { + if (fs::exists(output_folder, ec) && !fs::is_empty(output_folder, ec)) + add(Severity::Error, + "The output folder '" + output_folder.string() + "' already contains files. Choose an empty or new folder."); + + fs::path existing = fs::absolute(output_folder, ec); + while (!existing.empty() && !fs::exists(existing, ec) && existing != existing.parent_path()) + existing = existing.parent_path(); + const auto space = fs::space(existing, ec); + if (ec) + add(Severity::Warning, "Could not determine the free disk space at '" + existing.string() + "'."); + else if (space.available < r.estimated_disk_bytes) + add(Severity::Error, + "Not enough free disk space for the output folder: " + need + " needed, " + formatBytes(space.available) + + " free. Free some space or choose a folder on another disk."); + else if (space.available < r.estimated_disk_bytes + r.estimated_disk_bytes / 2) + add(Severity::Warning, "Free disk space is tight: " + need + " needed, " + formatBytes(space.available) + " free."); + else + disk_summary = "Disk space: " + need + " needed, " + formatBytes(space.available) + " free."; + } + if (!disk_summary.empty() && r.ok()) + r.findings.push_back({ Severity::Ok, disk_summary }); + + if (r.ok()) + { + std::ostringstream summary; + summary.precision(1); + summary << std::fixed << "Recording looks good: " << f.pairs << " file pairs, " << r.total_points / 1000000.0 + << " million points, " << r.duration_s / 60.0 << " minutes."; + r.findings.insert(r.findings.begin(), { Severity::Ok, summary.str() }); + } + return r; + } +} // namespace easy_gui diff --git a/apps/easy_gui/src/preflight.h b/apps/easy_gui/src/preflight.h new file mode 100644 index 00000000..18624cc3 --- /dev/null +++ b/apps/easy_gui/src/preflight.h @@ -0,0 +1,76 @@ +#pragma once + +#include +#include +#include +#include + +namespace easy_gui +{ + enum class Severity + { + Ok, + Warning, + Error + }; + + struct Finding + { + Severity severity; + std::string message; // plain language, shown as-is in the GUI + }; + + //! The recording's files, classified exactly as load_data() in lidar_odometry.cpp does it (sorted names; every + //! .laz/.las is a point cloud, every .csv an IMU file; status*/cam0*/cam1* JSON ignored; the first other .json or + //! .mjc is the calibration; the first .sn is used). load_data pairs point clouds and IMU files *by position* and + //! only loads min(#laz, #csv) pairs, so `pairs` is what step 1 will actually read. + struct RecordingFiles + { + std::vector laz_files; + std::vector csv_files; + std::filesystem::path sn_file; + std::filesystem::path calibration_file; + size_t pairs = 0; + }; + + RecordingFiles classifyRecordingFiles(const std::filesystem::path& recording); + + struct PreflightResult + { + RecordingFiles files; + std::vector findings; + uint64_t total_points = 0; // point records in the paired LAZ files (incl. no-return points) + double duration_s = 0.0; // IMU time span + int lidar_id = -1; // the single lidar id (LAZ user_data) seen in the samples + uint64_t raw_laz_bytes = 0; // size of the paired LAZ files + uint64_t estimated_work_bytes = 0; // temporary, freed after the run + uint64_t estimated_result_bytes = 0; // kept: step 1 session + exported point cloud and trajectory + uint64_t estimated_disk_bytes = 0; // work + results: what must be free when the run starts + + bool ok() const + { + for (const auto& f : findings) + if (f.severity == Severity::Error) + return false; + return true; + } + }; + + //! Checks the recording and the output folder before anything is started. Catches, with a plain message, the + //! cases where step 1 would otherwise abort() (unreadable LAZ), exit() (calibration mismatch) or silently produce + //! something incomplete (unpaired files, non-overlapping clocks). + PreflightResult runPreflight(const std::filesystem::path& recording, const std::filesystem::path& output_folder); + + //! Temporary disk space of a run: step 1's caches (original + decimated + concatenated chunk points, ~45 B/point + //! each with decimation 0.01). Measured peak on a 117 M-point forest walk: ~108 B per raw point; 120 B leaves + //! some margin. + uint64_t estimateWorkBytes(uint64_t total_points); + + //! Disk space of what is kept: the step 1 session (scan_lio LAZ, ~0.85x the raw LAZ size measured) plus the + //! exported LAZ (1.4 PF6 with extra bytes and the near points, ~1.5x raw assumed until M2 measures it) and the + //! trajectory CSV (small). + uint64_t estimateResultBytes(uint64_t raw_laz_bytes); + + //! True if `inner` is `outer` or lies inside it (both made absolute and normalised first). + bool isSameOrInside(const std::filesystem::path& inner, const std::filesystem::path& outer); +} // namespace easy_gui diff --git a/apps/easy_gui/src/progress_parser.cpp b/apps/easy_gui/src/progress_parser.cpp new file mode 100644 index 00000000..d80b36b0 --- /dev/null +++ b/apps/easy_gui/src/progress_parser.cpp @@ -0,0 +1,209 @@ +#include "progress_parser.h" + +#include +#include + +namespace easy_gui +{ + namespace + { + struct KnownFailure + { + const char* marker; + const char* message; + }; + + // Lines that mean step 1 is going to fail (or already has), with what to tell the user. + const KnownFailure kKnownFailures[] = { + { "MISSING CALIBRATION", + "The calibration file does not match this scanner, so the recording cannot be processed with it. Remove the calibration " + "file from the recording, or use the one that belongs to this scanner." }, + { "WRONG CALIBRATION FILE", + "The calibration file belongs to a different scanner. Remove it from the recording, or use the one that belongs to this " + "scanner." }, + { "System is out of memory", + "The computer ran out of memory. Close other programs and try again, or process the recording on a computer with more " + "memory." }, + { "Loading poses went wrong", + "No trajectory could be computed from the IMU data. The IMU files are probably empty or damaged." }, + { "DLL ERROR", + "A point cloud file could not be read (damaged, or not fully copied from the scanner). Copy the recording again." }, + { "points_global spill", "Writing temporary data failed. The disk is probably full: free some space and try again." }, + { "changed or vanished since it was first read", + "A recording file changed or disappeared during processing (was the USB stick removed?). Keep the recording available " + "until processing has finished." }, + { "Lidar odometry failed", + "Lidar odometry did not succeed on this recording. Check that the scanner was moving steadily and that the recording " + "is complete; the details below show where it stopped." }, + }; + + const std::regex kFilesTotal(R"(Loading point clouds \((\d+) files\))"); + const std::regex kPointsKept(R"(^TOTAL: (\d+))"); + const std::regex kPrepare(R"(running iterations: (\d+)/(\d+))"); + const std::regex kOdometry(R"(finished optimizing .* - (\d+) / (\d+) at iteration)"); + const std::regex kSavedScan(R"(saving to: .*scan_lio_(\d+)\.laz)"); + + int toInt(const std::ssub_match& m) + { + try + { + return std::stoi(m.str()); + } catch (...) + { + return 0; + } + } + + // save_result writes one scan_lio file per 51 odometry chunks (its "counter > 50" grouping). + int expectedScans(int chunks) + { + return std::max(1, (chunks + 50) / 51); + } + } // namespace + + const char* phaseLabel(Step1Phase phase) + { + switch (phase) + { + case Step1Phase::Starting: + return "Starting"; + case Step1Phase::LoadingFiles: + return "Reading the recording"; + case Step1Phase::ImuTrajectory: + return "Computing the motion from the IMU"; + case Step1Phase::Preparing: + return "Preparing lidar odometry"; + case Step1Phase::Odometry: + return "Lidar odometry"; + case Step1Phase::Saving: + return "Saving the odometry result"; + case Step1Phase::Finished: + return "Lidar odometry finished"; + } + return ""; + } + + double Step1Progress::phaseFraction() const + { + auto ratio = [](int done, int total) + { + return total > 0 ? std::clamp(static_cast(done) / total, 0.0, 1.0) : -1.0; + }; + switch (phase) + { + case Step1Phase::LoadingFiles: + return ratio(files_done, files_total); + case Step1Phase::Preparing: + return ratio(prepare_done, prepare_total); + case Step1Phase::Odometry: + return ratio(chunks_done, chunks_total); + case Step1Phase::Saving: + return ratio(scans_saved, expectedScans(chunks_total)); + case Step1Phase::Finished: + return 1.0; + default: + return -1.0; + } + } + + double Step1Progress::overallFraction() const + { + // Share of the run time per phase, measured on a 10-minute forest walk (115 file pairs, 93 M points kept). + struct Span + { + Step1Phase phase; + double weight; + }; + static const Span kSpans[] = { + { Step1Phase::Starting, 0.0 }, { Step1Phase::LoadingFiles, 0.04 }, { Step1Phase::ImuTrajectory, 0.01 }, + { Step1Phase::Preparing, 0.05 }, { Step1Phase::Odometry, 0.85 }, { Step1Phase::Saving, 0.05 }, + }; + if (phase == Step1Phase::Finished) + return 1.0; + double done = 0.0; + for (const auto& span : kSpans) + { + if (span.phase == phase) + { + const double f = phaseFraction(); + return done + span.weight * (f < 0.0 ? 0.0 : f); + } + done += span.weight; + } + return done; + } + + void ProgressParser::advance(Step1Phase phase) + { + // Phases only move forward: a stray late line from an earlier phase must not move the bar backwards. + if (static_cast(phase) > static_cast(progress_.phase)) + progress_.phase = phase; + } + + void ProgressParser::fail(const std::string& reason, const std::string& line) + { + progress_.last_error_line = line; + if (!progress_.failed) // the first recognised cause is the real one; later lines are consequences + { + progress_.failed = true; + progress_.failure = reason; + } + } + + void ProgressParser::feed(const std::string& line) + { + for (const auto& known : kKnownFailures) + if (line.find(known.marker) != std::string::npos) + { + fail(known.message, line); + return; + } + if (line.find("[error]") != std::string::npos || line.find("ERROR") != std::string::npos) + progress_.last_error_line = line; + + std::smatch m; + if (line.find("Loading data..") != std::string::npos) + advance(Step1Phase::LoadingFiles); + else if (std::regex_search(line, m, kFilesTotal)) + { + advance(Step1Phase::LoadingFiles); + progress_.files_total = toInt(m[1]); + } + else if (line.find("counter_filtered_points:") != std::string::npos) + progress_.files_done++; // phaseFraction() clamps + else if (std::regex_search(line, m, kPointsKept)) + { + try + { + progress_.points_kept = std::stoll(m[1].str()); + } catch (...) + { + } + } + else if (line.find("start calculating trajectory") != std::string::npos) + advance(Step1Phase::ImuTrajectory); + else if (std::regex_search(line, m, kPrepare)) + { + advance(Step1Phase::Preparing); + progress_.prepare_done = toInt(m[1]); + progress_.prepare_total = toInt(m[2]); + } + else if (line.find("step_1_done") != std::string::npos) + progress_.prepare_done = progress_.prepare_total; + else if (std::regex_search(line, m, kOdometry)) + { + advance(Step1Phase::Odometry); + progress_.chunks_done = std::max(progress_.chunks_done, toInt(m[1])); + progress_.chunks_total = toInt(m[2]); + } + else if (std::regex_search(line, m, kSavedScan)) + { + advance(Step1Phase::Saving); + progress_.scans_saved = std::max(progress_.scans_saved, toInt(m[1]) + 1); + } + else if (line.find("original_points_to_save_cache_file_name") != std::string::npos) + advance(Step1Phase::Saving); + else if (line.find("remove cache:") != std::string::npos && line.find("FINISHED") != std::string::npos) + advance(Step1Phase::Finished); + } +} // namespace easy_gui diff --git a/apps/easy_gui/src/progress_parser.h b/apps/easy_gui/src/progress_parser.h new file mode 100644 index 00000000..b08c0c07 --- /dev/null +++ b/apps/easy_gui/src/progress_parser.h @@ -0,0 +1,63 @@ +#pragma once + +#include + +namespace easy_gui +{ + //! Phases of the lidar_odometry_step_1 child process, in the order they occur. + enum class Step1Phase + { + Starting, + LoadingFiles, // reading the LAZ files (load_data) + ImuTrajectory, // AHRS / initial trajectory from the IMU (calculate_trajectory) + Preparing, // cutting the trajectory into odometry chunks (compute_step_1) + Odometry, // the lidar odometry itself (compute_step_2) + Saving, // save_result + Finished + }; + + const char* phaseLabel(Step1Phase phase); + + struct Step1Progress + { + Step1Phase phase = Step1Phase::Starting; + int files_total = 0; + int files_done = 0; + int prepare_done = 0; + int prepare_total = 0; + int chunks_done = 0; + int chunks_total = 0; + int scans_saved = 0; + long long points_kept = 0; // "TOTAL: n" after loading: points left after the load filter + + bool failed = false; + std::string failure; // plain-language reason, if one was recognised + std::string last_error_line; // last raw line that looked like an error, for the details view + + //! Fraction 0..1 within the current phase, or -1 if the phase gives no measurable progress. + double phaseFraction() const; + //! Rough fraction 0..1 of the whole step 1 run (odometry dominates the run time). + double overallFraction() const; + }; + + //! Turns lidar_odometry_step_1's console output into progress. + //! + //! NOTE: this parses free-form log text that step 1 does not treat as an interface. It only uses a few stable + //! markers (see the unit test, which feeds an excerpt of a real log), never fails on unknown lines, and the GUI + //! falls back to an indeterminate bar if a phase never reports numbers. Parallel file loading can interleave + //! lines, so per-file progress counts markers instead of relying on line order. + class ProgressParser + { + public: + void feed(const std::string& line); + const Step1Progress& progress() const + { + return progress_; + } + + private: + void advance(Step1Phase phase); + void fail(const std::string& reason, const std::string& line); + Step1Progress progress_; + }; +} // namespace easy_gui diff --git a/apps/easy_gui/src/settings.h b/apps/easy_gui/src/settings.h new file mode 100644 index 00000000..be9a293d --- /dev/null +++ b/apps/easy_gui/src/settings.h @@ -0,0 +1,24 @@ +#pragma once + +#include + +namespace easy_gui +{ + //! User-adjustable settings. Everything else comes from the forestry preset. + struct Settings + { + //! Near points (horizontal range below the 1.0 m load filter) are exported only from this 3D range outwards. + //! 0.4 m: on measured raw data the operator's body/handle shows up only as small blobs inside 0.4 m. + double near_min_range_m = 0.4; + static constexpr double kNearMinRangeLowest = 0.1; + static constexpr double kNearMinRangeHighest = 0.95; // must stay below the 1.0 m load filter + + //! Keep the step 1 staging/preview folders after a successful run (for debugging). + bool keep_work_files = false; + + void clamp() + { + near_min_range_m = std::clamp(near_min_range_m, kNearMinRangeLowest, kNearMinRangeHighest); + } + }; +} // namespace easy_gui diff --git a/apps/easy_gui/src/staging.cpp b/apps/easy_gui/src/staging.cpp new file mode 100644 index 00000000..84017c44 --- /dev/null +++ b/apps/easy_gui/src/staging.cpp @@ -0,0 +1,69 @@ +#include "staging.h" + +namespace fs = std::filesystem; + +namespace easy_gui +{ + bool createStagingFolder(const RecordingFiles& files, const fs::path& staging, std::string& method, std::string& error) + { + std::error_code ec; + if (fs::exists(staging, ec)) + { + error = "The work folder '" + staging.string() + "' already exists."; + return false; + } + if (!fs::create_directories(staging, ec)) + { + error = "Could not create the work folder '" + staging.string() + "' (" + ec.message() + ")."; + return false; + } + + std::vector sources; + for (size_t i = 0; i < files.pairs; i++) + { + sources.push_back(files.laz_files[i]); + sources.push_back(files.csv_files[i]); + } + if (!files.sn_file.empty()) + sources.push_back(files.sn_file); + if (!files.calibration_file.empty()) + sources.push_back(files.calibration_file); + + enum class Method + { + Symlink, + Hardlink, + Copy + }; + Method m = Method::Symlink; + + for (const auto& source : sources) + { + const fs::path absolute_source = fs::absolute(source, ec); + const fs::path target = staging / source.filename(); + for (;;) + { + ec.clear(); + if (m == Method::Symlink) + fs::create_symlink(absolute_source, target, ec); + else if (m == Method::Hardlink) + fs::create_hard_link(absolute_source, target, ec); + else + fs::copy_file(absolute_source, target, ec); + if (!ec) + break; + if (m == Method::Copy) + { + error = "Could not prepare '" + source.filename().string() + "' for processing (" + ec.message() + ")."; + return false; + } + // Fall back for this and all following files; anything half-created is overwritten by the next try. + fs::remove(target, ec); + m = (m == Method::Symlink) ? Method::Hardlink : Method::Copy; + } + } + + method = (m == Method::Symlink) ? "symbolic links" : (m == Method::Hardlink) ? "hard links" : "copies"; + return true; + } +} // namespace easy_gui diff --git a/apps/easy_gui/src/staging.h b/apps/easy_gui/src/staging.h new file mode 100644 index 00000000..b2a022db --- /dev/null +++ b/apps/easy_gui/src/staging.h @@ -0,0 +1,20 @@ +#pragma once + +#include "preflight.h" + +#include +#include + +namespace easy_gui +{ + //! Fills `staging` (created, must not exist yet) with links to the files step 1 will read: the paired point cloud + //! and IMU files, the .sn file and the calibration file. + //! + //! Why: load_data() writes its cache/ and preview/ folders next to its input files, and the recording may be on + //! the scanner's USB stick. Step 1 runs on the staging folder instead, so nothing is ever written into the + //! recording. Only the paired files are linked, which is exactly what load_data would load anyway. + //! + //! Tries a symbolic link per file, then a hard link, then a copy (Windows without developer mode cannot create + //! symlinks; hard links only work on the same volume). `method` reports which one was used. + bool createStagingFolder(const RecordingFiles& files, const std::filesystem::path& staging, std::string& method, std::string& error); +} // namespace easy_gui diff --git a/apps/easy_gui/tests/CMakeLists.txt b/apps/easy_gui/tests/CMakeLists.txt new file mode 100644 index 00000000..5c2085b6 --- /dev/null +++ b/apps/easy_gui/tests/CMakeLists.txt @@ -0,0 +1,26 @@ +cmake_minimum_required(VERSION 4.0.0) + +project(easy_gui_tests) + +# Unit tests for apps/easy_gui's easy_gui_core (everything but the window). +# Doctest, like the other suites. The pre-flight and staging tests write small +# synthetic Mandeye recordings to the temp directory; the progress parser test +# replays an excerpt of a real lidar_odometry_step_1 log. +add_executable(easy_gui_tests + test_main.cpp + test_child_process.cpp + test_pipeline_integration.cpp + test_preflight.cpp + test_progress_parser.cpp + test_staging_and_preset.cpp +) + +target_include_directories(easy_gui_tests PRIVATE ${THIRDPARTY_DIRECTORY}/doctest) +target_link_libraries(easy_gui_tests PRIVATE easy_gui_core) + +if (MSVC) + target_compile_options(easy_gui_tests PRIVATE /bigobj) +endif() + +include(CTest) +add_test(NAME easy_gui_tests COMMAND easy_gui_tests) diff --git a/apps/easy_gui/tests/test_child_process.cpp b/apps/easy_gui/tests/test_child_process.cpp new file mode 100644 index 00000000..c5668532 --- /dev/null +++ b/apps/easy_gui/tests/test_child_process.cpp @@ -0,0 +1,76 @@ +#include + +#include "child_process.h" + +#include +#include +#include + +using easy_gui::ChildProcess; + +#ifndef _WIN32 // uses /bin/sh; the Windows implementation is exercised by running Easy GUI + +TEST_CASE("stdout and stderr are merged line by line and the exit code is returned") +{ + ChildProcess child; + std::string error; + REQUIRE(child.start("/bin/sh", { "-c", "echo one; echo two 1>&2; printf 'three\\r\\nfour'; exit 3" }, error)); + std::vector lines; + std::string line; + while (child.readLine(line)) + lines.push_back(line); + CHECK(child.wait() == 3); + REQUIRE(lines.size() == 4); + CHECK(lines[0] == "one"); + CHECK(lines[1] == "two"); + CHECK(lines[2] == "three"); // '\r' removed + CHECK(lines[3] == "four"); // last line without a newline +} + +TEST_CASE("stdin is closed, so a program waiting for input does not hang") +{ + // load_data does std::cin.get() before std::exit on a calibration mismatch. + ChildProcess child; + std::string error; + REQUIRE(child.start("/bin/sh", { "-c", "read x; echo got-eof" }, error)); + std::string line; + REQUIRE(child.readLine(line)); + CHECK(line == "got-eof"); + CHECK(child.wait() != -1); +} + +TEST_CASE("kill stops the child and its process group promptly") +{ + ChildProcess child; + std::string error; + // The background sleep is a grandchild: it must die too, or it would hold the pipe open. + REQUIRE(child.start("/bin/sh", { "-c", "sleep 30 & echo started; sleep 30" }, error)); + std::string line; + REQUIRE(child.readLine(line)); + CHECK(line == "started"); + + const auto t0 = std::chrono::steady_clock::now(); + std::thread killer( + [&] + { + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + child.kill(); + }); + while (child.readLine(line)) + { + } + killer.join(); + CHECK(child.wait() == -1); + CHECK(child.wasKilled()); + CHECK(std::chrono::steady_clock::now() - t0 < std::chrono::seconds(5)); +} + +TEST_CASE("starting a missing program fails with a message") +{ + ChildProcess child; + std::string error; + CHECK_FALSE(child.start("/nonexistent/program", {}, error)); + CHECK_FALSE(error.empty()); +} + +#endif diff --git a/apps/easy_gui/tests/test_helpers.h b/apps/easy_gui/tests/test_helpers.h new file mode 100644 index 00000000..ec73fdcd --- /dev/null +++ b/apps/easy_gui/tests/test_helpers.h @@ -0,0 +1,97 @@ +#pragma once + +#include + +#include +#include +#include +#include +#include +#include + +namespace easy_gui_test +{ + //! Fresh empty directory under the system temp dir, removed again in the destructor. + struct TempDir + { + std::filesystem::path path; + TempDir() + { + static std::atomic counter{ 0 }; + path = std::filesystem::temp_directory_path() / + ("easy_gui_test_" + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count()) + "_" + + std::to_string(counter++)); + std::filesystem::create_directories(path); + } + ~TempDir() + { + std::error_code ec; + std::filesystem::remove_all(path, ec); + } + }; + + //! Mandeye-style raw LAZ (LAS 1.2 PF1): `n` points starting at gps_time `t0` [s], 1 ms apart. + inline bool writeRawLaz(const std::filesystem::path& file, int n, double t0, int lidar_id = 0) + { + laszip_POINTER writer = nullptr; + if (laszip_create(&writer)) + return false; + laszip_header* header = nullptr; + laszip_get_header_pointer(writer, &header); + header->version_major = 1; + header->version_minor = 2; + header->point_data_format = 1; + header->point_data_record_length = 28; + header->x_scale_factor = header->y_scale_factor = header->z_scale_factor = 0.0001; + if (laszip_open_writer(writer, file.string().c_str(), 1)) + return false; + laszip_point* point = nullptr; + laszip_get_point_pointer(writer, &point); + for (int i = 0; i < n; i++) + { + point->gps_time = t0 + i * 0.001; + point->user_data = static_cast(lidar_id); + point->intensity = 10; + laszip_F64 xyz[3] = { 2.0 + i * 0.01, 1.0, 0.5 }; + laszip_set_coordinates(writer, xyz); + laszip_write_point(writer); + laszip_update_inventory(writer); + } + laszip_close_writer(writer); + laszip_destroy(writer); + return true; + } + + //! Mandeye IMU csv with named columns, `n` samples at 200 Hz starting at `t0` [s] (written in ns). + inline void writeImuCsv(const std::filesystem::path& file, int n, double t0) + { + std::ofstream out(file); + out << "timestamp gyroX gyroY gyroZ accX accY accZ imuId timestampUnix\n"; + for (int i = 0; i < n; i++) + { + const auto ns = static_cast((t0 + i * 0.005) * 1e9); + out << ns << " 0 0 0 0 0 1 0 " << ns / 1000000 << "\n"; + } + } + + inline void writeText(const std::filesystem::path& file, const std::string& text) + { + std::ofstream out(file, std::ios::binary); + out << text; + } + + //! A plausible two-pair recording (plus an unmatched trailing lidar file, like real recordings often have), + //! 60 s long so it does not trigger the short-recording warning. + inline void writeRecording(const std::filesystem::path& dir, double t0 = 1783328629.0) + { + std::filesystem::create_directories(dir); + writeRawLaz(dir / "lidar0000.laz", 2000, t0); + writeImuCsv(dir / "imu0000.csv", 6000, t0); + writeRawLaz(dir / "lidar0001.laz", 2000, t0 + 30.0); + writeImuCsv(dir / "imu0001.csv", 6000, t0 + 30.0); + writeRawLaz(dir / "lidar0002.laz", 100, t0 + 60.0); + writeText(dir / "lidar0000.sn", "0 ARMCP1U0038132\n"); + writeText(dir / "lidar0001.sn", "0 ARMCP1U0038132\n"); + writeText(dir / "status0000.json", "{}"); + } +} // namespace easy_gui_test diff --git a/apps/easy_gui/tests/test_main.cpp b/apps/easy_gui/tests/test_main.cpp new file mode 100644 index 00000000..b8e3a4bf --- /dev/null +++ b/apps/easy_gui/tests/test_main.cpp @@ -0,0 +1,2 @@ +#define DOCTEST_CONFIG_IMPLEMENT_WITH_MAIN +#include diff --git a/apps/easy_gui/tests/test_pipeline_integration.cpp b/apps/easy_gui/tests/test_pipeline_integration.cpp new file mode 100644 index 00000000..a079374e --- /dev/null +++ b/apps/easy_gui/tests/test_pipeline_integration.cpp @@ -0,0 +1,100 @@ +#include + +#include "pipeline.h" +#include "test_helpers.h" + +#include +#include +#include + +using namespace easy_gui; +namespace fs = std::filesystem; + +// Integration tests against the real lidar_odometry_step_1 (built next to this test binary). They need a real +// Mandeye recording, so they only run when EASY_GUI_TEST_RECORDING points to one (a few file pairs are enough). +namespace +{ + const char* testRecording() + { + return std::getenv("EASY_GUI_TEST_RECORDING"); + } + + PipelineStatus waitUntil(const Pipeline& pipeline, bool (*done)(const PipelineStatus&), std::chrono::seconds timeout) + { + const auto t0 = std::chrono::steady_clock::now(); + PipelineStatus s = pipeline.status(); + while (!done(s) && std::chrono::steady_clock::now() - t0 < timeout) + { + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + s = pipeline.status(); + } + return s; + } +} // namespace + +TEST_CASE("cancel stops a real step 1 run during odometry and cleans up" * doctest::skip(testRecording() == nullptr)) +{ + easy_gui_test::TempDir tmp; + const fs::path out = tmp.path / "out"; + Pipeline pipeline; + std::string error; + REQUIRE(pipeline.start(PipelineRequest{ testRecording(), out, findStep1Executable(), Settings{} }, error)); + + const PipelineStatus during = waitUntil( + pipeline, + [](const PipelineStatus& s) + { + return s.state != PipelineState::Running || (s.step1.phase == Step1Phase::Odometry && s.step1.chunks_done > 0); + }, + std::chrono::seconds(300)); + REQUIRE(during.state == PipelineState::Running); + REQUIRE(during.step1.phase == Step1Phase::Odometry); + + const auto t_cancel = std::chrono::steady_clock::now(); + pipeline.cancel(); + const PipelineStatus after = waitUntil( + pipeline, + [](const PipelineStatus& s) + { + return s.state != PipelineState::Running; + }, + std::chrono::seconds(30)); + CHECK(after.state == PipelineState::Cancelled); + CHECK(std::chrono::steady_clock::now() - t_cancel < std::chrono::seconds(10)); + CHECK_FALSE(fs::exists(out / "work")); // links, step 1 cache and partial session are gone + CHECK(fs::exists(out / "easy_gui.log")); // the log stays + CHECK(fs::exists(fs::path(testRecording()))); // and the recording, of course +} + +TEST_CASE("a recording too short for odometry fails with a message, not a crash" * doctest::skip(testRecording() == nullptr)) +{ + // One file pair of real data, cut to its first 200 points: step 1 starts, finds no usable data and must fail + // gracefully through Easy GUI. + easy_gui_test::TempDir tmp; + const fs::path rec = tmp.path / "rec"; + fs::create_directories(rec); + const fs::path source(testRecording()); + fs::copy_file(source / "imu0000.csv", rec / "imu0000.csv"); + fs::copy_file(source / "lidar0000.sn", rec / "lidar0000.sn"); + // Timestamps inside the IMU range, so the pre-flight check passes and step 1 really runs on unusable data. + std::ifstream imu(rec / "imu0000.csv"); + std::string header, first_sample; + std::getline(imu, header); + std::getline(imu, first_sample); + const double imu_t0 = std::stod(first_sample.substr(0, first_sample.find(' '))) / 1e9; + easy_gui_test::writeRawLaz(rec / "lidar0000.laz", 200, imu_t0 + 1.0); + + Pipeline pipeline; + std::string error; + REQUIRE(pipeline.start(PipelineRequest{ rec, tmp.path / "out", findStep1Executable(), Settings{} }, error)); + const PipelineStatus s = waitUntil( + pipeline, + [](const PipelineStatus& st) + { + return st.state != PipelineState::Running; + }, + std::chrono::seconds(120)); + CHECK(s.state == PipelineState::Failed); + CHECK_FALSE(s.message.empty()); + MESSAGE("message shown to the user: " << s.message); +} diff --git a/apps/easy_gui/tests/test_preflight.cpp b/apps/easy_gui/tests/test_preflight.cpp new file mode 100644 index 00000000..79723855 --- /dev/null +++ b/apps/easy_gui/tests/test_preflight.cpp @@ -0,0 +1,137 @@ +#include + +#include "preflight.h" +#include "test_helpers.h" + +#include + +using namespace easy_gui; +using easy_gui_test::TempDir; +namespace fs = std::filesystem; + +namespace +{ + bool hasFinding(const PreflightResult& r, Severity s, const std::string& text) + { + return std::any_of( + r.findings.begin(), + r.findings.end(), + [&](const Finding& f) + { + return f.severity == s && f.message.find(text) != std::string::npos; + }); + } +} // namespace + +TEST_CASE("classifyRecordingFiles pairs like load_data") +{ + TempDir tmp; + easy_gui_test::writeRecording(tmp.path / "rec"); + const auto f = classifyRecordingFiles(tmp.path / "rec"); + CHECK(f.laz_files.size() == 3); + CHECK(f.csv_files.size() == 2); + CHECK(f.pairs == 2); + CHECK(f.sn_file.filename() == "lidar0000.sn"); + CHECK(f.calibration_file.empty()); // status*.json is not a calibration +} + +TEST_CASE("a plausible recording passes with a warning about the unmatched file") +{ + TempDir tmp; + easy_gui_test::writeRecording(tmp.path / "rec"); + const auto r = runPreflight(tmp.path / "rec", tmp.path / "out"); + CHECK(r.ok()); + CHECK(r.lidar_id == 0); + CHECK(r.total_points == 4000); + CHECK(r.duration_s == doctest::Approx(60.0).epsilon(0.01)); + CHECK(hasFinding(r, Severity::Ok, "Recording looks good")); + CHECK(hasFinding(r, Severity::Warning, "lidar0002.laz")); + + // Disk estimate: 120 B per raw point of work space + 2.5x the paired raw LAZ size of results, always reported. + const uint64_t laz_bytes = fs::file_size(tmp.path / "rec" / "lidar0000.laz") + fs::file_size(tmp.path / "rec" / "lidar0001.laz"); + CHECK(r.raw_laz_bytes == laz_bytes); + CHECK(r.estimated_work_bytes == 4000 * 120); + CHECK(r.estimated_result_bytes == laz_bytes * 5 / 2); + CHECK(r.estimated_disk_bytes == r.estimated_work_bytes + r.estimated_result_bytes); + CHECK(hasFinding(r, Severity::Ok, "Disk space: about")); +} + +TEST_CASE("errors are reported in plain language") +{ + TempDir tmp; + const fs::path rec = tmp.path / "rec"; + + SUBCASE("missing folder") + { + const auto r = runPreflight(tmp.path / "nope", tmp.path / "out"); + CHECK_FALSE(r.ok()); + CHECK(hasFinding(r, Severity::Error, "does not exist")); + } + SUBCASE("not a recording") + { + fs::create_directories(rec); + easy_gui_test::writeText(rec / "notes.txt", "hello"); + const auto r = runPreflight(rec, tmp.path / "out"); + CHECK_FALSE(r.ok()); + CHECK(hasFinding(r, Severity::Error, "does not look like a Mandeye recording")); + } + SUBCASE("damaged point cloud file") + { + easy_gui_test::writeRecording(rec); + easy_gui_test::writeText(rec / "lidar0001.laz", "this is not a laz file"); + const auto r = runPreflight(rec, tmp.path / "out"); + CHECK_FALSE(r.ok()); + CHECK(hasFinding(r, Severity::Error, "lidar0001.laz")); + } + SUBCASE("two lidars") + { + easy_gui_test::writeRecording(rec); + easy_gui_test::writeRawLaz(rec / "lidar0001.laz", 2000, 1783328659.0, 1); + const auto r = runPreflight(rec, tmp.path / "out"); + CHECK_FALSE(r.ok()); + CHECK(hasFinding(r, Severity::Error, "more than one lidar")); + } + SUBCASE("clocks do not overlap") + { + easy_gui_test::writeRecording(rec); + easy_gui_test::writeImuCsv(rec / "imu0000.csv", 6000, 1000.0); + easy_gui_test::writeImuCsv(rec / "imu0001.csv", 6000, 1030.0); + const auto r = runPreflight(rec, tmp.path / "out"); + CHECK_FALSE(r.ok()); + CHECK(hasFinding(r, Severity::Error, "do not overlap")); + } + SUBCASE("output inside the recording") + { + easy_gui_test::writeRecording(rec); + const auto r = runPreflight(rec, rec / "results"); + CHECK_FALSE(r.ok()); + CHECK(hasFinding(r, Severity::Error, "inside the recording folder")); + } + SUBCASE("output folder not empty") + { + easy_gui_test::writeRecording(rec); + fs::create_directories(tmp.path / "out"); + easy_gui_test::writeText(tmp.path / "out" / "old.txt", "x"); + const auto r = runPreflight(rec, tmp.path / "out"); + CHECK_FALSE(r.ok()); + CHECK(hasFinding(r, Severity::Error, "already contains files")); + } + SUBCASE("calibration for another scanner") + { + easy_gui_test::writeRecording(rec); + easy_gui_test::writeText( + rec / "calibration.json", R"({"calibration": {"OTHERSERIAL": {"identity": "true"}}, "imuToUse": "OTHERSERIAL"})"); + const auto r = runPreflight(rec, tmp.path / "out"); + CHECK_FALSE(r.ok()); + CHECK(hasFinding(r, Severity::Error, "different scanner")); + } +} + +TEST_CASE("isSameOrInside") +{ + CHECK(isSameOrInside("/a/b/c", "/a/b")); + CHECK(isSameOrInside("/a/b", "/a/b")); + CHECK(isSameOrInside("/a/b/", "/a/b")); + CHECK_FALSE(isSameOrInside("/a/bc", "/a/b")); + CHECK_FALSE(isSameOrInside("/a", "/a/b")); +} diff --git a/apps/easy_gui/tests/test_progress_parser.cpp b/apps/easy_gui/tests/test_progress_parser.cpp new file mode 100644 index 00000000..cea44a56 --- /dev/null +++ b/apps/easy_gui/tests/test_progress_parser.cpp @@ -0,0 +1,157 @@ +#include + +#include "progress_parser.h" + +#include +#include + +using namespace easy_gui; + +namespace +{ + // Excerpt of a real lidar_odometry_step_1 CLI log (forest walk, 115 file pairs), paths shortened. The order and + // the interleaving of the per-file lines (parallel loading) are as captured. + const std::vector kRealLogStart = { + "SUCCESS: Version compatible: 0.105.0", + "Parameters loaded OK from: /out/work/easy_gui_step1.toml", + "\"/out/work/staging/lidar0080.laz\"", + "Loading data..", + "--------------", + "Only using first SN file found", + "There is no calibration file in folder!", + "IGNORE THIS MESSAGE IF YOU ONLY HAVE 1 LiDAR", + "loading IMU data from 'imu****.csv' using ID 0 from reference '/out/work/staging/lidar0000.sn' ..", + "Loading point clouds (115 files)..", + "total - filter = kept", + "-----------------------", + "header->number_of_point_records: 1008288", + "counter_filtered_points: 141284", + "1008288 - 141284 = 867004", + "header->number_of_point_records: 1002336", + "counter_filtered_points: 184105", + "1002336 - 184105 = 818231", + "counter_filtered_points: 189379", // its header line was interleaved away + "1000704 - 189379 = 811325", + }; + + const std::vector kRealLogMiddle = { + "TOTAL: 93371048 (lazy: ranges recorded, clouds loaded on demand)", + "..loading finished.", + "start calculating trajectory..", + "AHRS: Fusion (gain=0.5, samples=115959)", + "number of poses: 114944", + "Creating folder: \"/out/work/staging/cache\"", + "running iterations: 1/5747", + "running iterations: 2851/5747", + "[2026-10-17 18:40:42.920] [info] finished optimizing (since start : 0.100 [s]) - 1 / 5747 at iteration 172 / 500 in (load " + "vector 0.012 [s], step1 0.000 [s], step2 0.000 [s], odometry 0.066 [s]) with acc_distance 0.000[m], delta<1.0e-12 " + "(converged)", + "[2026-10-17 18:47:02.538] [info] finished optimizing (since start : 379.716 [s]) - 1810 / 5747 at iteration 182 / 500 in " + "(load vector 0.003 [s], step1 0.000 [s], step2 0.000 [s], odometry 0.195 [s]) with acc_distance 157.907[m], delta<1.0e-12 " + "(converged)", + }; +} // namespace + +TEST_CASE("phases and numbers from a real log") +{ + ProgressParser parser; + CHECK(parser.progress().phase == Step1Phase::Starting); + CHECK(parser.progress().phaseFraction() < 0.0); + + for (const auto& line : kRealLogStart) + parser.feed(line); + CHECK(parser.progress().phase == Step1Phase::LoadingFiles); + CHECK(parser.progress().files_total == 115); + CHECK(parser.progress().files_done == 3); + CHECK(parser.progress().phaseFraction() == doctest::Approx(3.0 / 115.0)); + + for (size_t i = 0; i < 4; i++) + parser.feed(kRealLogMiddle[i]); + CHECK(parser.progress().points_kept == 93371048); + CHECK(parser.progress().phase == Step1Phase::ImuTrajectory); + CHECK(parser.progress().phaseFraction() < 0.0); // no numbers: the GUI shows an indeterminate bar + + for (size_t i = 4; i < 8; i++) + parser.feed(kRealLogMiddle[i]); + CHECK(parser.progress().phase == Step1Phase::Preparing); + CHECK(parser.progress().prepare_done == 2851); + CHECK(parser.progress().prepare_total == 5747); + + parser.feed(kRealLogMiddle[8]); + parser.feed(kRealLogMiddle[9]); + CHECK(parser.progress().phase == Step1Phase::Odometry); + CHECK(parser.progress().chunks_done == 1810); + CHECK(parser.progress().chunks_total == 5747); + const double overall = parser.progress().overallFraction(); + CHECK(overall > 0.10); + CHECK(overall < 0.50); + CHECK_FALSE(parser.progress().failed); +} + +TEST_CASE("saving and finishing") +{ + ProgressParser parser; + parser.feed("[2026-10-17 18:40:42.920] [info] finished optimizing (since start : 1.0 [s]) - 102 / 102 at iteration 1 / 500 in x"); + parser.feed("original_points_to_save_cache_file_name \"/out/work/staging/cache/session_chunk_0.cache\""); + CHECK(parser.progress().phase == Step1Phase::Saving); + parser.feed("saving to: \"/out/work/lio_session/scan_lio_0.laz\""); + CHECK(parser.progress().scans_saved == 1); + CHECK(parser.progress().phaseFraction() == doctest::Approx(0.5)); // 102 chunks -> 2 scan files + parser.feed("remove cache: '/out/work/staging/cache' FINISHED"); + CHECK(parser.progress().phase == Step1Phase::Finished); + CHECK(parser.progress().overallFraction() == doctest::Approx(1.0)); +} + +TEST_CASE("phases never go backwards") +{ + ProgressParser parser; + parser.feed("[x] [info] finished optimizing (since start : 1.0 [s]) - 5 / 10 at iteration 1 / 500 in x"); + parser.feed("running iterations: 1/10"); // a late line from an earlier phase + CHECK(parser.progress().phase == Step1Phase::Odometry); +} + +TEST_CASE("known failures become plain-language messages") +{ + struct Case + { + const char* line; + const char* expected_fragment; + }; + const Case cases[] = { + { "MISSING CALIBRATION FOR imuSnToUse: ABC!!!", "calibration file does not match" }, + { "WRONG CALIBRATION FILE! THE SERIAL NUMBER SHOULD BE X!!!", "different scanner" }, + { "System is out of memory : std::bad_alloc", "ran out of memory" }, + { "Loading poses went wrong! Could not load poses!", "IMU" }, + { "[2026-10-17] [error] points_global spill: write failed", "disk is probably full" }, + { "lazy_load_raw_clouds: '/x/lidar0001.laz' changed or vanished since it was first read", "USB stick" }, + { "Lidar odometry failed: no results saved", "did not succeed" }, + }; + for (const auto& c : cases) + { + CAPTURE(c.line); + ProgressParser parser; + parser.feed(c.line); + CHECK(parser.progress().failed); + CHECK(parser.progress().failure.find(c.expected_fragment) != std::string::npos); + CHECK(parser.progress().last_error_line == c.line); + } + + SUBCASE("the first cause wins") + { + ProgressParser parser; + parser.feed("MISSING CALIBRATION FOR imuSnToUse: ABC!!!"); + parser.feed("Lidar odometry failed: no results saved"); + CHECK(parser.progress().failure.find("calibration") != std::string::npos); + } +} + +TEST_CASE("unknown and malformed lines are ignored") +{ + ProgressParser parser; + parser.feed(""); + parser.feed("running iterations: x/y"); + parser.feed("finished optimizing - a / b at iteration"); + parser.feed("Loading point clouds (lots files).."); + CHECK(parser.progress().phase == Step1Phase::Starting); + CHECK_FALSE(parser.progress().failed); +} diff --git a/apps/easy_gui/tests/test_staging_and_preset.cpp b/apps/easy_gui/tests/test_staging_and_preset.cpp new file mode 100644 index 00000000..eb46ab35 --- /dev/null +++ b/apps/easy_gui/tests/test_staging_and_preset.cpp @@ -0,0 +1,66 @@ +#include + +#include "forestry_preset.h" +#include "staging.h" +#include "test_helpers.h" + +#include + +#include + +using namespace easy_gui; +using easy_gui_test::TempDir; +namespace fs = std::filesystem; + +TEST_CASE("staging links exactly the files step 1 reads and never touches the recording") +{ + TempDir tmp; + const fs::path rec = tmp.path / "rec"; + easy_gui_test::writeRecording(rec); + const auto files = classifyRecordingFiles(rec); + + std::string method, error; + REQUIRE(createStagingFolder(files, tmp.path / "staging", method, error)); + CHECK(fs::exists(tmp.path / "staging" / "lidar0000.laz")); + CHECK(fs::exists(tmp.path / "staging" / "imu0001.csv")); + CHECK(fs::exists(tmp.path / "staging" / "lidar0000.sn")); + CHECK_FALSE(fs::exists(tmp.path / "staging" / "lidar0002.laz")); // unpaired: load_data would skip it anyway + CHECK_FALSE(fs::exists(tmp.path / "staging" / "status0000.json")); + + // The staged view reads the same data. + CHECK(fs::file_size(tmp.path / "staging" / "lidar0001.laz") == fs::file_size(rec / "lidar0001.laz")); + + // Removing the staging folder (as the pipeline does) leaves the recording intact. + fs::remove_all(tmp.path / "staging"); + CHECK(fs::exists(rec / "lidar0000.laz")); + CHECK(fs::exists(rec / "imu0000.csv")); + + SUBCASE("an existing staging folder is refused") + { + fs::create_directories(tmp.path / "staging2"); + CHECK_FALSE(createStagingFolder(files, tmp.path / "staging2", method, error)); + } +} + +TEST_CASE("the step 1 parameter file is preset 2 plus the output-only filter change") +{ + TempDir tmp; + std::string error; + REQUIRE(writeStep1ParameterFile(tmp.path / "p.toml", error)); + + auto params = std::make_unique(); + TomlIO toml_io; + REQUIRE(toml_io.LoadParametersFromTomlFile((tmp.path / "p.toml").string(), *params)); + + CHECK(params->decimation == doctest::Approx(0.01)); + CHECK(params->in_out_params_indoor.resolution_X == doctest::Approx(0.1)); + CHECK(params->in_out_params_outdoor.resolution_Z == doctest::Approx(0.3)); + CHECK(params->filter_threshold_xy_inner == doctest::Approx(1.0)); + CHECK(params->filter_threshold_xy_outer == doctest::Approx(70.0)); + CHECK(params->nr_iter == 500); + CHECK(params->sliding_window_trajectory_length_threshold == doctest::Approx(10000.0)); + CHECK(params->max_distance_lidar == doctest::Approx(70.0)); + CHECK_FALSE(params->use_robust_and_accurate_lidar_odometry); + // The preset says 1.5; Easy GUI lowers it to the load filter (output-only, see forestry_preset.h). + CHECK(params->threshould_output_filter == doctest::Approx(params->filter_threshold_xy_inner)); +} From 41a2f3475eb04252fb0af528d259348a3e0499ea Mon Sep 17 00:00:00 2001 From: Martin Mokros Date: Sat, 17 Oct 2026 20:22:21 +0100 Subject: [PATCH 02/23] Add Easy GUI export (M2): LAS 1.4 point cloud with near points, trajectory CSV After lidar odometry, Easy GUI now writes point_cloud.laz, trajectory.csv and a README.txt describing every field into the output folder. - point_cloud.laz: LAS 1.4, point format 6, gps_time in Unix epoch seconds (global encoding bit 0 = 0), point_source_id = scan index, user_data = lidar id, named extra bytes range_m (float32) and near_added (uint8). The extra-bytes VLR is written by hand: laszip_add_attribute always sets the scale bit, which makes readers return near_added as float64. - Odometry points are placed exactly as step 2's export does it (Session in load_cache_mode for poses and trajectories, points streamed per scan): on a 117 M-point forest walk all 89,326,756 points are integer-identical to multi_view_tls_registration_step_2 --export; the trajectory matches step 2's trajectories.csv to 0.05 mm. - Near points: raw points step 1's distance filters removed (the exact complement of its load and output filters), at least 0.4 m away, no-return points dropped, placed with step 1's own rule (last node before the raw time, within 10 ms). A --check-band test mode re-adds the 1.0-1.2 m band step 1 also exported: NN distance to step 1's points median 0.10 mm, p99 12.9 mm where step 1 has the point; rebuilding step 1's timestamp ramp reproduces its points to 0.17 mm and attributes the remaining difference to the ramp time choosing a neighbouring pose. - Streams one scan or one raw file at a time: peak memory 102 MB, ~1.5 min on the full recording. Measured sizes set the pre-flight result estimate to 2.25x the raw LAZ size (session 0.82x + point cloud 1.24x + CSV 0.03x). - New Export stage with progress and cancel in the window and in --headless; an export failure keeps the odometry session. --export-only runs the export on an existing session. - core_no_gui is now always built (Easy GUI links it for Session). - Tests: LAS 1.4 writer round trip, the export on a synthetic step 1 session with known poses (near rule edge cases, check band, cancel), and an end-to-end run against the real step 1. scripts/check_export.py runs the acceptance checks (fields, range, alignment, step 2 reference) with laspy. Co-Authored-By: Claude Opus 5.5 --- apps/easy_gui/CMakeLists.txt | 7 + apps/easy_gui/scripts/check_export.py | 228 ++++++++ apps/easy_gui/src/easy_gui_main.cpp | 152 ++++- apps/easy_gui/src/export.cpp | 547 ++++++++++++++++++ apps/easy_gui/src/export.h | 76 +++ apps/easy_gui/src/las14_writer.cpp | 208 +++++++ apps/easy_gui/src/las14_writer.h | 73 +++ apps/easy_gui/src/pipeline.cpp | 78 ++- apps/easy_gui/src/pipeline.h | 15 +- apps/easy_gui/src/preflight.cpp | 5 +- apps/easy_gui/src/preflight.h | 5 +- apps/easy_gui/tests/CMakeLists.txt | 4 +- apps/easy_gui/tests/test_export.cpp | 498 ++++++++++++++++ .../tests/test_pipeline_integration.cpp | 27 + apps/easy_gui/tests/test_preflight.cpp | 4 +- core/CMakeLists.txt | 18 +- 16 files changed, 1903 insertions(+), 42 deletions(-) create mode 100644 apps/easy_gui/scripts/check_export.py create mode 100644 apps/easy_gui/src/export.cpp create mode 100644 apps/easy_gui/src/export.h create mode 100644 apps/easy_gui/src/las14_writer.cpp create mode 100644 apps/easy_gui/src/las14_writer.h create mode 100644 apps/easy_gui/tests/test_export.cpp diff --git a/apps/easy_gui/CMakeLists.txt b/apps/easy_gui/CMakeLists.txt index 5a6cec41..e9555d99 100644 --- a/apps/easy_gui/CMakeLists.txt +++ b/apps/easy_gui/CMakeLists.txt @@ -14,11 +14,17 @@ project(easy_gui) # - lidar_odometry_utils.cpp for MLvxCalib (the pre-flight check mirrors # load_data's calibration checks with step 1's own parser), # - toml_io.cpp to write the parameter file with step 1's own TomlIO. +# toml_io.cpp also reads the parameters back from the session for the export. +# The export (src/export.cpp) loads the step 1 session with core's Session in +# load_cache_mode (poses + trajectories exactly as step 2 sees them, no points) +# and streams the points itself, so core_no_gui is linked below. # WITH_GUI=0 is PUBLIC because LidarOdometryParams changes layout under # WITH_GUI (clear_color), so every TU that sees it must agree. add_library(easy_gui_core STATIC src/child_process.cpp src/child_process.h + src/export.cpp src/export.h src/forestry_preset.cpp src/forestry_preset.h + src/las14_writer.cpp src/las14_writer.h src/pipeline.cpp src/pipeline.h src/preflight.cpp src/preflight.h src/progress_parser.cpp src/progress_parser.h @@ -50,6 +56,7 @@ target_include_directories(easy_gui_core # needs (the same reason step 1's own tests link core_no_gui); TBB and laszip for # its load_point_cloud()/decimate(), which the pre-flight check never calls. target_link_libraries(easy_gui_core PUBLIC + core_no_gui core_math vqf Fusion diff --git a/apps/easy_gui/scripts/check_export.py b/apps/easy_gui/scripts/check_export.py new file mode 100644 index 00000000..1bbf7a8c --- /dev/null +++ b/apps/easy_gui/scripts/check_export.py @@ -0,0 +1,228 @@ +#!/usr/bin/env python3 +"""Acceptance checks for an Easy GUI export (point_cloud.laz + trajectory.csv). + + check_export.py [--step2 ] + +Needs laspy with a LAZ backend, numpy and scipy (pip install "laspy[lazrs]" numpy scipy). + +Checks: + fields LAS 1.4 / PF6, GPS time type bit, named extra bytes (range_m float32, near_added uint8), + point_source_id = scan index, user_data = lidar id, gps_time is Unix seconds. + range histogram of range_m; nothing below the near limit, nothing at the sensor origin. + alignment only with an export made with --check-band: nearest-neighbour distance from each check-band point + (raw point placed the near-point way) to step 1's own points. A few mm median = same transform chain. + reference with --step2: the near_added = 0 points against step 2's full-resolution export of the same session + (multi_view_tls_registration_step_2 --export ): count, XYZ, intensity, trajectory. +Exit code 1 if a check fails. +""" + +import argparse +import sys +from pathlib import Path + +import laspy +import numpy as np +from scipy.spatial import cKDTree + +CHUNK = 5_000_000 +failures = [] + + +def check(ok, message): + print((" ok " if ok else " FAIL ") + message) + if not ok: + failures.append(message) + + +def check_fields(laz): + print("fields") + with laspy.open(laz) as f: + h = f.header + check(str(h.version) == "1.4" and h.point_format.id == 6, f"LAS {h.version}, point format {h.point_format.id}") + check(not h.global_encoding.gps_time_type, "global encoding bit 0 (GPS time type) = 0") + extra = {d.name: d for d in h.point_format.extra_dimensions} + check(set(extra) == {"range_m", "near_added"}, f"extra bytes: {sorted(extra)}") + check(np.dtype(extra["range_m"].dtype) == np.float32, f"range_m is {np.dtype(extra['range_m'].dtype)}") + check(np.dtype(extra["near_added"].dtype) == np.uint8, f"near_added is {np.dtype(extra['near_added'].dtype)}") + print(f" {h.point_count:,} points, bounds {h.mins.round(1)} .. {h.maxs.round(1)}, {h.generating_software}") + + +def scan(laz): + """One streaming pass: per-flag counts, range histogram, sensor-origin and time checks, and the points needed for + the alignment check (step 1 points up to 3 m + check-band points), all with bounded memory. 3 m, not the band's + 1.2 m: the band limits the horizontal range, steep points reach ~2 m in 3D.""" + stats = {"count": np.zeros(3, np.int64), "hist": np.zeros(1201, np.int64), "range_min": [np.inf] * 3, + "range_max": [-np.inf] * 3, "t_min": np.inf, "t_max": -np.inf, "psid": set(), "user_data": set(), + "origin": 0, "close_odo": [], "band": [], "big_range": []} + with laspy.open(laz) as f: + for p in f.chunk_iterator(CHUNK): + flag = np.asarray(p.near_added) + r = np.asarray(p.range_m) + xyz = np.column_stack([p.x, p.y, p.z]) + for k in range(3): + m = flag == k + if m.any(): + stats["count"][k] += m.sum() + stats["range_min"][k] = min(stats["range_min"][k], float(r[m].min())) + stats["range_max"][k] = max(stats["range_max"][k], float(r[m].max())) + stats["hist"] += np.bincount(np.minimum((r / 0.1).astype(np.int64), 1200), minlength=1201) + t = np.asarray(p.gps_time) + stats["t_min"], stats["t_max"] = min(stats["t_min"], t.min()), max(stats["t_max"], t.max()) + stats["psid"].update(np.unique(np.asarray(p.point_source_id)).tolist()) + stats["user_data"].update(np.unique(np.asarray(p.user_data)).tolist()) + stats["origin"] += int((np.abs(xyz).sum(axis=1) == 0).sum()) + int((r == 0).sum()) + m = (flag == 0) & (r < 3.0) + stats["close_odo"].append((xyz[m], r[m])) + stats["band"].append((xyz[flag == 2], np.asarray(p.gps_time)[flag == 2])) + far = r > 70 + if far.any(): + stats["big_range"].append(np.column_stack([r[far], xyz[far, 2], np.asarray(p.point_source_id)[far]])) + return stats + + +def check_range(stats, near_min): + print("range") + names = ["odometry (0)", "near added (1)", "check band (2)"] + for k in range(3): + if stats["count"][k]: + print(f" {names[k]:15s} {stats['count'][k]:>12,} points, range_m {stats['range_min'][k]:.3f} .. {stats['range_max'][k]:.3f} m") + check(min(stats["range_min"]) >= near_min - 1e-6, f"no range_m below {near_min} m (min {min(stats['range_min']):.4f})") + check(stats["origin"] == 0, f"no points at (0,0,0) or with range 0 ({stats['origin']})") + h = stats["hist"] + edges = [0, 0.4, 1.0, 1.5, 3, 10, 30, 50, 70, 80, 100, 120.1] + print(" range_m histogram:") + for a, b in zip(edges[:-1], edges[1:]): + n = h[int(round(a * 10)):int(round(b * 10))].sum() + print(f" {a:6.1f} - {b:6.1f} m {n:>12,} {100 * n / h.sum():6.2f} %") + beyond = h[700:].sum() + print(f" beyond 70 m: {beyond:,} points ({100 * beyond / h.sum():.4f} %)") + if stats["big_range"]: + # step 1's 70 m limit is horizontal, in the lidar frame: steep returns can be further away in 3D + big = np.concatenate(stats["big_range"]) + print(f" (step 1 limits the horizontal range only; these are in {len(np.unique(big[:, 2]))} scans, " + f"z {big[:, 1].min():.1f} .. {big[:, 1].max():.1f} m)") + + +def check_alignment(stats, folder): + band = np.concatenate([b[0] for b in stats["band"]]) if stats["band"] else np.empty((0, 3)) + if len(band) == 0: + return + print("alignment (check-band points vs step 1 points)") + band_t = np.concatenate([b[1] for b in stats["band"]]) + band_r_max = stats["range_max"][2] + odo = np.concatenate([o[0][o[1] <= band_r_max + 0.01] for o in stats["close_odo"]]) + d, _ = cKDTree(odo).query(band, k=1, workers=-1) + d_mm = d * 1000 + # Step 1 drops the raw points between the last pose of each 20-pose worker chunk and the next chunk's first pose + # (~5 % of the time); check-band points from there have no step 1 counterpart, so they are reported separately. + traj = np.genfromtxt(folder / "trajectory.csv", delimiter=",", names=True, usecols=(0, 11)) + node_in_scan = np.zeros(len(traj), np.int64) + for s in np.unique(traj["scan_index"]): + m = np.where(traj["scan_index"] == s)[0] + node_in_scan[m] = np.arange(len(m)) + node = np.searchsorted(traj["timestamp_unix_s"], band_t, side="left") - 1 + in_gap = node_in_scan[node] % 20 == 19 + + def pct(x): + q = np.percentile(x, [50, 90, 99]) + return f"median {q[0]:.2f} mm, p90 {q[1]:.2f}, p99 {q[2]:.2f}, max {x.max():.1f} mm" + + print(f" {len(band):,} check-band points vs {len(odo):,} step 1 points up to {band_r_max:.2f} m") + print(f" all: {pct(d_mm)}") + print(f" with counterpart: {pct(d_mm[~in_gap])} ({(~in_gap).sum():,} points)") + print(f" in chunk gaps: {pct(d_mm[in_gap])} ({in_gap.sum():,} points, no step 1 counterpart)") + print(f" exactly coincident (< 0.15 mm, same trajectory pose as step 1): {100 * (d_mm[~in_gap] < 0.15).mean():.1f} %") + check(np.median(d_mm) < 5.0, f"median NN distance {np.median(d_mm):.2f} mm < 5 mm") + + +def check_reference(laz, step2_dir): + print("reference (near_added = 0 vs step 2 export)") + ref_file = step2_dir / "all_step_2.laz" + n_ref = laspy.open(ref_file).header.point_count + with laspy.open(laz) as f, laspy.open(ref_file) as g: + it_ref = g.chunk_iterator(CHUNK) + pending = None + n = 0 + max_d = np.zeros(3) + intensity_diff = 0 + psid_diff = 0 + gps_scale = None + for p in f.chunk_iterator(CHUNK): + m = np.asarray(p.near_added) == 0 + if not m.any(): + continue + ours = np.column_stack([p.x[m], p.y[m], p.z[m]]) + ours_i = np.asarray(p.intensity)[m] + ours_t = np.asarray(p.gps_time)[m] + ours_s = np.asarray(p.point_source_id)[m] + k = 0 + while k < len(ours): + if pending is None or len(pending[0]) == 0: + q = next(it_ref) + pending = (np.column_stack([q.x, q.y, q.z]), np.asarray(q.intensity), np.asarray(q.gps_time), + np.asarray(q.point_source_id)) + take = min(len(ours) - k, len(pending[0])) + max_d = np.maximum(max_d, np.abs(ours[k:k + take] - pending[0][:take]).max(axis=0)) + intensity_diff += int((ours_i[k:k + take] != pending[1][:take]).sum()) + psid_diff += int((ours_s[k:k + take] != pending[3][:take]).sum()) + if gps_scale is None: + gps_scale = pending[2][0] / ours_t[k] + pending = tuple(a[take:] for a in pending) + k += take + n += len(ours) + check(n == n_ref, f"point count {n:,} (step 2: {n_ref:,})") + check(max_d.max() <= 0.0002, f"max |dXYZ| {max_d * 1000} mm <= 0.2 mm") + check(intensity_diff == 0, f"intensity identical ({intensity_diff} differ)") + check(psid_diff == 0, f"point_source_id identical ({psid_diff} differ)") + print(f" step 2 gps_time / ours = {gps_scale:.6g} (step 2 writes the session's ns times x 1e9)") + + +def check_trajectory(folder, step2_dir, stats): + print("trajectory") + traj = np.genfromtxt(folder / "trajectory.csv", delimiter=",", names=True) + print(f" {len(traj):,} poses, {traj['timestamp_unix_s'][0]:.6f} .. {traj['timestamp_unix_s'][-1]:.6f} s") + check(np.all(np.diff(traj["timestamp_unix_s"]) > 0), "timestamps strictly increasing") + check(traj["timestamp_unix_s"][0] <= stats["t_min"] + 0.02 and traj["timestamp_unix_s"][-1] >= stats["t_max"] - 0.02, + "trajectory spans the point times") + q = np.column_stack([traj["qw"], traj["qx"], traj["qy"], traj["qz"]]) + check(np.abs(np.linalg.norm(q, axis=1) - 1).max() < 1e-6, "unit quaternions") + check(set(np.unique(traj["scan_index"]).astype(int).tolist()) == stats["psid"], "scan_index matches point_source_id") + if step2_dir: + ref = np.loadtxt(step2_dir / "trajectories.csv", delimiter=",") + check(len(ref) == len(traj), f"pose count {len(traj):,} (step 2: {len(ref):,})") + dt = np.abs(ref[:, 0] / 1e9 - traj["timestamp_unix_s"]).max() + dxyz = np.abs(ref[:, 2:5] - np.column_stack([traj["x"], traj["y"], traj["z"]])).max() + # step 2: qx qy qz qw; q and -q are the same rotation + rq = ref[:, [8, 5, 6, 7]] + dq = np.minimum(np.abs(rq - q).max(axis=1), np.abs(rq + q).max(axis=1)).max() + check(dt < 1e-6, f"timestamps match step 2 (max diff {dt * 1e6:.3f} us)") + check(dxyz <= 0.00005, f"positions match step 2 (max diff {dxyz * 1000:.3f} mm)") + check(dq < 1e-8, f"orientations match step 2 (max quaternion diff {dq:.2e})") + host = np.abs(ref[:, 1] / 1e3 - traj["host_clock_unix_s"]).max() + check(host < 1e-3, f"host clock matches step 2's Unix column / 1000 (max diff {host:.4f} s)") + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("folder", type=Path) + ap.add_argument("--step2", type=Path) + ap.add_argument("--near-min", type=float, default=0.4) + a = ap.parse_args() + laz = a.folder / "point_cloud.laz" + check_fields(laz) + stats = scan(laz) + check(stats["user_data"] and len(stats["user_data"]) == 1, f"user_data (lidar id) values {sorted(stats['user_data'])}") + print(f" point_source_id {min(stats['psid'])} .. {max(stats['psid'])} ({len(stats['psid'])} scans)") + check(1.5e9 < stats["t_min"] < stats["t_max"] < 2.2e9, f"gps_time in Unix seconds ({stats['t_min']:.3f} .. {stats['t_max']:.3f})") + check_range(stats, a.near_min) + check_alignment(stats, a.folder) + if a.step2: + check_reference(laz, a.step2) + check_trajectory(a.folder, a.step2, stats) + check((a.folder / "README.txt").is_file(), "README.txt present") + print(f"\n{'ALL CHECKS PASSED' if not failures else f'{len(failures)} CHECK(S) FAILED'}") + return 1 if failures else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/apps/easy_gui/src/easy_gui_main.cpp b/apps/easy_gui/src/easy_gui_main.cpp index 84f1073e..20c7be84 100644 --- a/apps/easy_gui/src/easy_gui_main.cpp +++ b/apps/easy_gui/src/easy_gui_main.cpp @@ -1,6 +1,7 @@ // HDMapping Easy GUI: raw Mandeye recording -> lidar odometry (precision forestry preset) -> point cloud + trajectory. // See CLAUDE.md ("Easy GUI") for the decisions behind the pipeline. +#include "export.h" #include "pipeline.h" #include "preflight.h" #include "settings.h" @@ -13,12 +14,15 @@ #include #include +#include +#include #include #include #include #include #include #include +#include namespace fs = std::filesystem; using namespace easy_gui; @@ -253,18 +257,18 @@ namespace return; } - // Stage list. Stage indices: 0 check+prepare, 1 odometry, 2 export (M2). + // Stage list. Stage indices: 0 check+prepare, 1 odometry, 2 export, 3 = all done. const bool failed = s.state == PipelineState::Failed; int current = 0; - if (s.stage == PipelineStage::Odometry || s.stage == PipelineStage::Cleanup) + if (s.stage == PipelineStage::Odometry) current = 1; - if (s.state == PipelineState::Succeeded) + if (s.stage == PipelineStage::Export || (s.stage == PipelineStage::Cleanup && !s.session_folder.empty())) current = 2; + if (s.state == PipelineState::Succeeded) + current = 3; stageRow("Check and prepare the recording", 0, current, failed && current == 0); stageRow("Lidar odometry (precision forestry settings)", 1, current, failed && current == 1); - ImGui::TextColored(kGrey, "[ ]"); - ImGui::SameLine(); - ImGui::TextDisabled("Export point cloud and trajectory (coming in the next version)"); + stageRow("Export point cloud and trajectory", 2, current, failed && current == 2); if (s.state == PipelineState::Running) { @@ -288,6 +292,12 @@ namespace ImGui::TextUnformatted("Overall"); ImGui::ProgressBar(static_cast(overall), ImVec2(-1, 0), eta.c_str()); } + else if (s.stage == PipelineStage::Export) + { + ImGui::TextUnformatted(s.export_step.empty() ? "Exporting..." : s.export_step.c_str()); + ImGui::ProgressBar(static_cast(s.export_fraction), ImVec2(-1, 0)); + ImGui::TextUnformatted(("Elapsed " + formatDuration(s.elapsed_s)).c_str()); + } else { ImGui::TextUnformatted(s.stage == PipelineStage::Cleanup ? "Cleaning up..." : "Preparing..."); @@ -315,7 +325,20 @@ namespace ImGui::Spacing(); if (s.state == PipelineState::Succeeded) + { wrappedColored(kGreen, s.message + " (took " + formatDuration(s.elapsed_s) + ")"); + const OutputLayout out(app.output_folder); + const ExportStats& e = s.export_stats; + ImGui::BulletText( + "%s: %.1f million points (%.1f million from lidar odometry, %.1f million near points), %.0f MB", + out.point_cloud.filename().string().c_str(), + (e.odometry_points + e.near_points + e.check_band_points) / 1e6, + e.odometry_points / 1e6, + e.near_points / 1e6, + e.laz_bytes / 1e6); + ImGui::BulletText("%s: %zu scanner poses", out.trajectory.filename().string().c_str(), e.trajectory_nodes); + ImGui::BulletText("%s describes every field of both files.", out.readme.filename().string().c_str()); + } else if (s.state == PipelineState::Cancelled) wrappedColored(kAmber, s.message); else @@ -358,7 +381,7 @@ namespace { //! `easy_gui --headless `: the same pipeline without a window, for scripted runs //! (the dev validation script) and remote machines. Exit code 0 = success, 1 = failed / refused, 2 = usage. - int runHeadless(const fs::path& recording, const fs::path& output_folder) + int runHeadless(const fs::path& recording, const fs::path& output_folder, double check_band_outer_m) { // Unbuffered, so progress shows up as it happens even when redirected to a file or pipe (MSVC treats _IOLBF as // full buffering); the output is a line every few seconds. @@ -376,7 +399,7 @@ namespace Pipeline pipeline; std::string error; - if (!pipeline.start(PipelineRequest{ recording, output_folder, findStep1Executable(), Settings{} }, error)) + if (!pipeline.start(PipelineRequest{ recording, output_folder, findStep1Executable(), Settings{}, check_band_outer_m }, error)) { std::printf("[X] %s\n", error.c_str()); return 1; @@ -387,13 +410,16 @@ namespace const PipelineStatus s = pipeline.status(); char line[256]; const double f = s.step1.phaseFraction(); - std::snprintf( - line, - sizeof(line), - "%s: %s overall %.0f%%", - phaseLabel(s.step1.phase), - f >= 0.0 ? (std::to_string(static_cast(f * 100)) + "%,").c_str() : "", - s.step1.overallFraction() * 100.0); + if (s.stage == PipelineStage::Export) + std::snprintf(line, sizeof(line), "Export: %.0f%%", s.export_fraction * 100.0); + else + std::snprintf( + line, + sizeof(line), + "%s: %s overall %.0f%%", + phaseLabel(s.step1.phase), + f >= 0.0 ? (std::to_string(static_cast(f * 100)) + "%,").c_str() : "", + s.step1.overallFraction() * 100.0); if (last != line) std::printf("[%s] %s\n", formatDuration(s.elapsed_s).c_str(), line); last = line; @@ -406,18 +432,100 @@ namespace std::printf(" detail: %s\n", s.detail.c_str()); return s.state == PipelineState::Succeeded ? 0 : 1; } + + //! Dev aid: `easy_gui --export-only [--check-band ]` runs only the + //! export, on an existing step 1 session of that recording (step 1 takes ~20 min on a 10-min walk; the export + //! about a minute). The output folder may exist; the three result files in it are overwritten. + int runExportOnly(const fs::path& recording, const fs::path& session_file, const fs::path& output_folder, double check_band_outer_m) + { + std::setvbuf(stdout, nullptr, _IONBF, 0); + const PreflightResult check = runPreflight(recording, output_folder); + if (check.files.pairs == 0) + { + std::printf("[X] no lidar/IMU file pairs in '%s'\n", recording.string().c_str()); + return 1; + } + const OutputLayout out(output_folder); + ExportRequest request; + request.session_file = session_file; + request.recording = check.files; + request.lidar_id = std::max(0, check.lidar_id); + request.near_min_range_m = Settings{}.near_min_range_m; + request.check_band_outer_m = check_band_outer_m; + request.laz_file = out.point_cloud; + request.trajectory_file = out.trajectory; + request.readme_file = out.readme; + + std::atomic cancel{ false }; + int last_percent = -1; + ExportStats stats; + std::string error; + const bool ok = exportSession( + request, + cancel, + [&](double fraction, const std::string& step) + { + const int percent = static_cast(fraction * 100); + if (percent / 10 != last_percent / 10) + std::printf("[%3d%%] %s\n", percent, step.c_str()); + last_percent = percent; + }, + stats, + error); + if (!ok) + { + std::printf("[X] %s\n", error.c_str()); + return 1; + } + std::printf( + "[OK] %llu odometry + %llu near + %llu check-band points; raw read %llu, no-return %llu, near outside trajectory %llu; " + "range %.3f - %.3f m; %zu poses; LAZ %llu bytes, CSV %llu bytes\n", + static_cast(stats.odometry_points), + static_cast(stats.near_points), + static_cast(stats.check_band_points), + static_cast(stats.raw_points_read), + static_cast(stats.raw_no_return), + static_cast(stats.raw_near_outside_trajectory), + stats.range_min_m, + stats.range_max_m, + stats.trajectory_nodes, + static_cast(stats.laz_bytes), + static_cast(stats.trajectory_bytes)); + return 0; + } + + //! Removes `--check-band ` from the arguments; returns the value, or 0 when absent. + double takeCheckBand(std::vector& args) + { + for (size_t i = 0; i + 1 < args.size(); i++) + if (args[i] == "--check-band") + { + const double value = std::atof(args[i + 1].c_str()); + args.erase(args.begin() + static_cast(i), args.begin() + static_cast(i) + 2); + return value; + } + return 0.0; + } } // namespace int main(int argc, char* argv[]) { - if (argc > 1 && std::string(argv[1]) == "--headless") + std::vector args(argv + 1, argv + argc); + if (!args.empty() && (args[0] == "--headless" || args[0] == "--export-only")) { - if (argc != 4) - { - std::printf("usage: %s --headless \n", argv[0]); - return 2; - } - return runHeadless(argv[2], argv[3]); + const double check_band = takeCheckBand(args); + if (args[0] == "--headless" && args.size() == 3) + return runHeadless(args[1], args[2], check_band); + if (args[0] == "--export-only" && args.size() == 4) + return runExportOnly(args[1], args[2], args[3], check_band); + std::printf( + "usage: %s --headless [--check-band ]\n" + " %s --export-only [--check-band ]\n" + "--check-band (testing): also export the raw points between the 1.0 m load filter and , flagged\n" + " near_added = 2, to check that near points line up with the odometry points.\n", + argv[0], + argv[0]); + return 2; } // Dev aid: `--screenshot [recording]` renders ~2 s, saves the window and exits (layout checks without diff --git a/apps/easy_gui/src/export.cpp b/apps/easy_gui/src/export.cpp new file mode 100644 index 00000000..760f9bf5 --- /dev/null +++ b/apps/easy_gui/src/export.cpp @@ -0,0 +1,547 @@ +#include "export.h" + +#include "las14_writer.h" + +#include +#include + +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; + +namespace easy_gui +{ + namespace + { + constexpr double kCoordinateScale = 0.0001; // as step 2's export (LazWriter), so XYZ quantise identically + //! A raw point belongs to the last trajectory node before it, if that node is at most this much older. The same + //! tolerance step 1 applies when it assigns points to nodes (compute_step_1). + constexpr double kNodeTolerance_s = 0.01; + constexpr double kRangeBin_m = 0.1; + + //! Extra bytes of every exported point, in this order. + const std::vector kAttributes = { + { ExtraBytesAttribute::Type::F32, "range_m", "3D distance from the lidar [m]" }, + { ExtraBytesAttribute::Type::U8, "near_added", "0 odometry, 1 near point added" }, + }; + +#pragma pack(push, 1) + struct ExtraBytes + { + float range_m; + uint8_t near_added; + }; +#pragma pack(pop) + static_assert(sizeof(ExtraBytes) == 5); + + //! Streaming laszip reader: one point at a time, positions with the header's scale and offset applied. + class LazReader + { + public: + ~LazReader() + { + if (reader_) + { + laszip_close_reader(reader_); + laszip_destroy(reader_); + } + } + + bool open(const fs::path& file, std::string& error) + { + laszip_BOOL compressed = 0; + if (laszip_create(&reader_) || laszip_open_reader(reader_, file.string().c_str(), &compressed) || + laszip_get_header_pointer(reader_, &header_) || laszip_get_point_pointer(reader_, &point_)) + { + error = "could not open '" + file.string() + "'"; + return false; + } + count_ = header_->number_of_point_records ? header_->number_of_point_records : header_->extended_number_of_point_records; + return true; + } + + uint64_t count() const + { + return count_; + } + + bool next(std::string& error) + { + if (laszip_read_point(reader_)) + { + error = "could not read a point"; + return false; + } + return true; + } + + Eigen::Vector3d position() const + { + return { header_->x_offset + header_->x_scale_factor * static_cast(point_->X), + header_->y_offset + header_->y_scale_factor * static_cast(point_->Y), + header_->z_offset + header_->z_scale_factor * static_cast(point_->Z) }; + } + + const laszip_point& point() const + { + return *point_; + } + + private: + laszip_POINTER reader_ = nullptr; + laszip_header* header_ = nullptr; + laszip_point* point_ = nullptr; + uint64_t count_ = 0; + }; + + //! One trajectory node with everything the export needs. + struct Node + { + double time_s; // lidar clock (Unix epoch seconds on Mandeye) + size_t scan; + Eigen::Affine3d global; // scan pose (session_poses.mrp) x node pose relative to the scan + }; + + void addRange(ExportStats& stats, double range) + { + const uint64_t total = stats.odometry_points + stats.near_points + stats.check_band_points; + if (total == 0 || range < stats.range_min_m) + stats.range_min_m = range; + if (total == 0 || range > stats.range_max_m) + stats.range_max_m = range; + const size_t bin = std::min(stats.range_histogram.size() - 1, static_cast(std::max(0.0, range) / kRangeBin_m)); + stats.range_histogram[bin]++; + } + + bool writeTrajectory( + const fs::path& file, const Session& session, const Eigen::Vector3d& offset, ExportStats& stats, std::string& error) + { + std::ofstream out(file); + if (!out) + { + error = "Could not create the trajectory file '" + file.string() + "'."; + return false; + } + out << "timestamp_unix_s,x,y,z,qw,qx,qy,qz,om_deg,fi_deg,ka_deg,scan_index,host_clock_unix_s,imu_om_deg,imu_fi_deg,imu_ka_" + "deg\n"; + const auto& clouds = session.point_clouds_container.point_clouds; + char line[512]; + for (size_t i = 0; i < clouds.size(); i++) + { + for (const auto& node : clouds[i].local_trajectory) + { + // As save_trajectories() in step 2: scan pose x node pose, plus the session offset. + Eigen::Affine3d pose = clouds[i].m_pose * node.m_pose; + pose.translation() += offset; + const Eigen::Quaterniond q(pose.rotation()); + const TaitBryanPose tb = pose_tait_bryan_from_affine_matrix(pose); + // node.timestamps.second is the IMU file's timestampUnix column: the scanner computer's clock in + // milliseconds (step 1 passes it through unchanged). + std::snprintf( + line, + sizeof(line), + "%.6f,%.4f,%.4f,%.4f,%.9f,%.9f,%.9f,%.9f,%.6f,%.6f,%.6f,%zu,%.3f,%.6f,%.6f,%.6f\n", + node.timestamps.first / 1e9, + pose.translation().x(), + pose.translation().y(), + pose.translation().z(), + q.w(), + q.x(), + q.y(), + q.z(), + rad2deg(tb.om), + rad2deg(tb.fi), + rad2deg(tb.ka), + i, + node.timestamps.second / 1e3, + rad2deg(node.imu_om_fi_ka.x()), + rad2deg(node.imu_om_fi_ka.y()), + rad2deg(node.imu_om_fi_ka.z())); + out << line; + stats.trajectory_nodes++; + } + } + out.close(); + if (!out) + { + error = "Could not write the trajectory file '" + file.string() + "' (is the disk full?)."; + return false; + } + return true; + } + + std::string formatCount(uint64_t n) + { + std::string s = std::to_string(n); + for (int i = static_cast(s.size()) - 3; i > 0; i -= 3) + s.insert(static_cast(i), ","); + return s; + } + + bool writeReadme(const ExportRequest& request, const ExportStats& stats, std::string& error) + { + std::ofstream out(request.readme_file); + if (!out) + { + error = "Could not create '" + request.readme_file.string() + "'."; + return false; + } + const std::string laz = request.laz_file.filename().string(); + const std::string csv = request.trajectory_file.filename().string(); + char inner[32], near_min[32], output_filter[32], outer[32]; + std::snprintf(inner, sizeof(inner), "%.2f m", stats.filter_threshold_xy_inner); + std::snprintf(outer, sizeof(outer), "%.1f m", stats.filter_threshold_xy_outer); + std::snprintf(near_min, sizeof(near_min), "%.2f m", request.near_min_range_m); + std::snprintf(output_filter, sizeof(output_filter), "%.2f m", stats.threshould_output_filter); + + out << "HDMapping Easy GUI results\n" + << "==========================\n\n" + << "Made by HDMapping Easy GUI " << HDMAPPING_CONFIGURE_PROJECT_VERSION << " from the recording\n" + << " " << request.recording.laz_files.front().parent_path().string() << "\n" + << "with lidar odometry (precision forestry settings). No loop closure and no georeferencing: coordinates\n" + << "are in metres in a local frame that starts at the scanner's first position (z up, from the IMU).\n\n" + << "Files\n-----\n" + << " " << laz << " point cloud (LAZ, LAS 1.4, point format 6), " + << formatCount(stats.odometry_points + stats.near_points + stats.check_band_points) << " points\n" + << " " << csv << " trajectory of the scanner (CSV), " << formatCount(stats.trajectory_nodes) << " poses\n" + << " easy_gui.log processing log\n" + << " work/lio_session/ lidar odometry session; open session.mjs in multi_view_tls_registration_step_2\n" + << " to refine or georeference the result\n\n" + << "Time\n----\n" + << "All times are Unix epoch seconds (UTC, seconds since 1970-01-01) as counted by the scanner's lidar\n" + << "clock. They are NOT GPS week time and NOT adjusted standard GPS time, even though the LAS field is called\n" + << "gps_time (the LAS header's global encoding bit 0 is 0 accordingly).\n" + << "Points from lidar odometry (near_added = 0) carry lidar odometry's time, not the raw measurement time:\n" + << "within each 0.1 s piece of the recording, odometry spreads its points' times evenly between the first and\n" + << "last pose of that piece. That is close to the measurement time (typically 1-4 ms off, at most about\n" + << "20 ms), but it cannot be used to match points to the raw files. Odometry also places each point with the\n" + << "pose at that time, so the same offset shows up as small position differences between odometry points\n" + << "and near points: usually a few millimetres, rarely more than 2 cm. Near points (near_added = 1) keep\n" + << "their raw measurement time.\n\n" + << "Point cloud fields (" << laz << ")\n" + << std::string(21 + laz.size(), '-') << "\n" + << " X, Y, Z position [m], 0.1 mm resolution\n" + << " intensity lidar reflectivity as recorded (Livox: 0-255; usually 0-160)\n" + << " gps_time time [Unix epoch seconds], see \"Time\"\n" + << " point_source_id index of the lidar odometry scan (scan_lio_.laz in work/lio_session, and\n" + << " scan_index in " << csv << "); about 5 s of recording each\n" + << " user_data lidar id (" << request.lidar_id << " for every point of this single-lidar recording)\n" + << " return_number, number_of_returns always 1; classification always 0 (not classified)\n" + << " range_m extra bytes, float32: 3D distance from the lidar at the moment of measurement [m];\n" + << " can exceed " << outer << " for steep points, as that limit is horizontal\n" + << " near_added extra bytes, uint8: 0 = point from lidar odometry, 1 = near point added by Easy GUI\n" + << (stats.check_band_points ? " 2 = TEST MODE alignment-check point (duplicate of an odometry point)\n" + : "") + << "\n" + << "Near points\n-----------\n" + << "Lidar odometry only uses and exports points whose horizontal distance from the lidar (in the lidar's own\n" + << "frame) is between " << inner << " and " << outer << " and whose 3D distance is above " << output_filter + << ". Easy GUI adds back\n" + << "the raw " + << "points those limits removed at close range, so the cloud has no hole around the walking path:\n" + << "a raw point is added (near_added = 1) when\n" + << " - its 3D distance from the lidar is at least " << near_min << " (closer returns are mostly the operator's\n" + << " body and the scanner handle; the limit is the advanced setting \"Closest points kept\"), and\n" + << " - lidar odometry left it out because of distance: horizontal distance at most " << inner << ", or 3D\n" + << " distance at most " << output_filter << ", and horizontal distance below " << outer << ", and\n" + << " - it is a real return: points at exactly (0, 0, 0) or with time 0 are \"no return\" records and are\n" + << " never exported.\n" + << "Each near point is placed with the trajectory pose lidar odometry uses for points measured at the same\n" + << "time (the last pose before the point), so it lines up with the odometry points. Points measured where\n" + << "the trajectory has no pose within 10 ms (start and end of the recording) are left out.\n" + << "Near points: " << formatCount(stats.near_points) << ", odometry points: " << formatCount(stats.odometry_points) + << ".\n\n" + << "Trajectory fields (" << csv << ", comma-separated, one header row)\n" + << std::string(53 + csv.size(), '-') << "\n" + << " timestamp_unix_s time of the pose [Unix epoch seconds], same clock as the point cloud's gps_time\n" + << " x, y, z position of the scanner (IMU) [m], same frame as the point cloud\n" + << " qw, qx, qy, qz orientation as a unit quaternion (rotates scanner coordinates into the frame)\n" + << " om_deg, fi_deg, ka_deg the same orientation as HDMapping's Tait-Bryan angles [deg]\n" + << " scan_index lidar odometry scan the pose belongs to (point_source_id in the point cloud)\n" + << " host_clock_unix_s the scanner computer's own clock at that IMU sample [Unix epoch seconds,\n" + << " millisecond resolution]; it is not synchronised with timestamp_unix_s\n" + << " imu_om_deg, imu_fi_deg, imu_ka_deg orientation from the IMU alone, before lidar odometry [deg]\n"; + out.close(); + if (!out) + { + error = "Could not write '" + request.readme_file.string() + "'."; + return false; + } + return true; + } + } // namespace + + fs::path findSessionParameterFile(const fs::path& session_folder) + { + std::error_code ec; + fs::path found; + for (const auto& entry : fs::directory_iterator(session_folder, ec)) + { + const std::string name = entry.path().filename().string(); + if (name.starts_with("HDMapping_params_") && entry.path().extension() == ".toml" && (found.empty() || entry.path() > found)) + found = entry.path(); + } + return found; + } + + bool exportSession( + const ExportRequest& request, + const std::atomic& cancel, + const ExportProgress& progress, + ExportStats& stats, + std::string& error) + { + stats = ExportStats{}; + const auto cancelled = [&]() + { + if (cancel) + error = "Export was cancelled."; + return cancel.load(); + }; + + // --- the parameters step 1 ran with: its distance filters define which raw points are "near points" + const fs::path parameter_file = findSessionParameterFile(request.session_file.parent_path()); + auto params = std::make_unique(); + TomlIO toml_io; + if (parameter_file.empty() || !toml_io.LoadParametersFromTomlFile(parameter_file.string(), *params)) + { + error = "The lidar odometry result is incomplete: its settings file (HDMapping_params_*.toml) is missing in '" + + request.session_file.parent_path().string() + "'."; + return false; + } + stats.filter_threshold_xy_inner = params->filter_threshold_xy_inner; + stats.filter_threshold_xy_outer = params->filter_threshold_xy_outer; + stats.threshould_output_filter = params->threshould_output_filter; + + // --- poses and trajectories exactly as step 2 loads them; the points are streamed below instead + progress(0.0, "Reading the lidar odometry result"); + Session session; + session.load_cache_mode = true; + if (!session.load(request.session_file.string(), false, 0.0, 0.0, 0.0, false) || + session.point_clouds_container.point_clouds.empty()) + { + error = "Could not read the lidar odometry result '" + request.session_file.string() + "'."; + return false; + } + const auto& clouds = session.point_clouds_container.point_clouds; + const Eigen::Vector3d offset = session.point_clouds_container.offset; + stats.scans = clouds.size(); + if (clouds.size() > 65535) + { + error = "The recording is too long for one point cloud file (more than 65535 lidar odometry scans)."; + return false; + } + + // --- calibration, as load_data builds it: lidar frame -> the frame step 1 works in (identity without a file) + Eigen::Affine3d calibration = Eigen::Affine3d::Identity(); + { + const auto id_to_sn = MLvxCalib::GetIdToSnMapping(request.recording.sn_file.string()); + const auto combined = MLvxCalib::CombineIntoCalibration( + id_to_sn, + request.recording.calibration_file.empty() + ? std::unordered_map{} + : MLvxCalib::GetCalibrationFromFile(request.recording.calibration_file.string())); + if (const auto it = combined.find(request.lidar_id); it != combined.end()) + calibration = it->second; + } + const Eigen::Affine3d lidar_from_sensor = calibration.inverse(); + + // --- all trajectory nodes in time order, for placing the near points + std::vector nodes; + for (size_t i = 0; i < clouds.size(); i++) + { + if (clouds[i].local_trajectory.empty()) + { + error = "The lidar odometry result is incomplete: scan " + std::to_string(i) + " has no trajectory."; + return false; + } + for (const auto& n : clouds[i].local_trajectory) + nodes.push_back({ n.timestamps.first / 1e9, i, clouds[i].m_pose * n.m_pose }); + } + std::stable_sort( + nodes.begin(), + nodes.end(), + [](const Node& a, const Node& b) + { + return a.time_s < b.time_s; + }); + std::vector node_times(nodes.size()); + for (size_t i = 0; i < nodes.size(); i++) + node_times[i] = nodes[i].time_s; + + // --- output files + std::error_code ec; + fs::create_directories(request.laz_file.parent_path(), ec); + Las14Writer writer; + if (!writer.open( + request.laz_file.string(), + kAttributes, + offset, + kCoordinateScale, + std::string("HDMapping ") + HDMAPPING_CONFIGURE_PROJECT_VERSION, + error)) + { + error = "Could not create the point cloud file '" + request.laz_file.string() + "' (" + error + ")."; + return false; + } + const uint8_t user_data = static_cast(request.lidar_id); + const double work_units = static_cast(clouds.size() + request.recording.pairs); + auto writePoint = [&](const Eigen::Vector3d& position, double time_s, uint16_t intensity, size_t scan, double range, NearFlag flag) + { + const ExtraBytes extra{ static_cast(range), flag }; + Las14Point p; + p.position = position; + p.gps_time = time_s; + p.intensity = intensity; + p.point_source_id = static_cast(scan); + p.user_data = user_data; + p.extra_bytes = reinterpret_cast(&extra); + addRange(stats, range); + (flag == kFromOdometry ? stats.odometry_points : flag == kNearAdded ? stats.near_points : stats.check_band_points)++; + return writer.write(p, error); + }; + + // --- 1. step 1's points, one scan at a time + for (size_t i = 0; i < clouds.size(); i++) + { + if (cancelled()) + return false; + progress( + static_cast(i) / work_units, + "Writing the point cloud (scan " + std::to_string(i + 1) + " of " + std::to_string(clouds.size()) + ")"); + + const auto& pc = clouds[i]; + std::vector times; // node times in step 1's unit (seconds x 1e9), as the points' gps_time + std::vector sensor_from_local; + for (const auto& n : pc.local_trajectory) + { + times.push_back(n.timestamps.first); + sensor_from_local.push_back(n.m_pose.inverse()); + } + + LazReader reader; + if (!reader.open(pc.file_name, error)) + { + error = "The lidar odometry result is incomplete: " + error + "."; + return false; + } + for (uint64_t k = 0; k < reader.count(); k++) + { + if (!reader.next(error)) + { + error = "Could not read '" + pc.file_name + "': " + error + "."; + return false; + } + const laszip_point& rp = reader.point(); + if (rp.gps_time == 0.0) // step 2's export skips these too + continue; + // Step 2 subtracts the session offset from the loaded points and adds it back after the pose. + const Eigen::Vector3d local = reader.position() - offset; + + // Range: the point in the sensor frame, through the node step 1 used for it (the last node before the + // point's time), then into the lidar frame. + const auto it = std::lower_bound(times.begin(), times.end(), rp.gps_time); + const size_t node = it == times.begin() ? 0 : static_cast(it - times.begin()) - 1; + const double range = (lidar_from_sensor * (sensor_from_local[node] * local)).norm(); + + if (!writePoint(pc.m_pose * local + offset, rp.gps_time / 1e9, rp.intensity, i, range, kFromOdometry)) + { + error = "Could not write the point cloud file (is the disk full?): " + error + "."; + return false; + } + } + } + + // --- 2. near points from the raw recording, one file at a time + const double inner = params->filter_threshold_xy_inner; + const double outer = params->filter_threshold_xy_outer; + const double output_filter = params->threshould_output_filter; + for (size_t f = 0; f < request.recording.pairs; f++) + { + if (cancelled()) + return false; + progress( + static_cast(clouds.size() + f) / work_units, + "Adding the near points (file " + std::to_string(f + 1) + " of " + std::to_string(request.recording.pairs) + ")"); + + const fs::path& raw_file = request.recording.laz_files[f]; + LazReader reader; + if (!reader.open(raw_file, error)) + { + error = "Could not read the recording file '" + raw_file.filename().string() + "'."; + return false; + } + for (uint64_t k = 0; k < reader.count(); k++) + { + if (!reader.next(error)) + { + error = "Could not read the recording file '" + raw_file.filename().string() + "': " + error + "."; + return false; + } + stats.raw_points_read++; + const laszip_point& rp = reader.point(); + if ((rp.X == 0 && rp.Y == 0 && rp.Z == 0) || rp.gps_time == 0.0) + { + stats.raw_no_return++; + continue; + } + const Eigen::Vector3d lidar = reader.position(); + const double range_xy = std::hypot(lidar.x(), lidar.y()); + const double range = lidar.norm(); + if (range < request.near_min_range_m || range_xy >= outer) + continue; + const Eigen::Vector3d sensor = calibration * lidar; + + // Exactly the complement of step 1's distance filters: load_point_cloud keeps + // inner < range_xy < outer, points_to_vector keeps |sensor| > threshould_output_filter. + NearFlag flag; + if (range_xy <= inner || sensor.norm() <= output_filter) + flag = kNearAdded; + else if (range_xy < request.check_band_outer_m) + flag = kCheckBand; + else + continue; + + const double t = rp.gps_time; + const auto it = std::lower_bound(node_times.begin(), node_times.end(), t); + if (it == node_times.begin() || t - *(it - 1) > kNodeTolerance_s) + { + stats.raw_near_outside_trajectory++; + continue; + } + const Node& node = nodes[static_cast(it - node_times.begin()) - 1]; + if (!writePoint(node.global * sensor + offset, t, rp.intensity, node.scan, range, flag)) + { + error = "Could not write the point cloud file (is the disk full?): " + error + "."; + return false; + } + } + } + + if (!writer.close(error)) + { + error = "Could not finish the point cloud file (is the disk full?): " + error + "."; + return false; + } + stats.laz_bytes = fs::file_size(request.laz_file, ec); + + progress(1.0, "Writing the trajectory"); + if (!writeTrajectory(request.trajectory_file, session, offset, stats, error)) + return false; + stats.trajectory_bytes = fs::file_size(request.trajectory_file, ec); + return writeReadme(request, stats, error); + } +} // namespace easy_gui diff --git a/apps/easy_gui/src/export.h b/apps/easy_gui/src/export.h new file mode 100644 index 00000000..b507cdec --- /dev/null +++ b/apps/easy_gui/src/export.h @@ -0,0 +1,76 @@ +#pragma once + +#include "preflight.h" + +#include +#include +#include +#include +#include +#include + +namespace easy_gui +{ + //! Values of near_added, the extra-bytes flag of every exported point. + enum NearFlag : uint8_t + { + kFromOdometry = 0, // exported by lidar odometry (step 1) + kNearAdded = 1, // raw point step 1's distance filters removed, added by Easy GUI + kCheckBand = 2, // test mode only: raw point step 1 also exported, added again to check the alignment + }; + + struct ExportRequest + { + std::filesystem::path session_file; // step 1 session.mjs + RecordingFiles recording; // the raw files step 1 read (near points come from them) + int lidar_id = 0; // user_data of the raw points (single-lidar recordings only) + double near_min_range_m = 0.4; // near points closer than this (3D) are left out + //! Test mode: > 0 also adds the raw points with filter_threshold_xy_inner < horizontal range < this, which step 1 + //! exported too, flagged kCheckBand, so their placement can be compared with step 1's own points. + double check_band_outer_m = 0.0; + std::filesystem::path laz_file; + std::filesystem::path trajectory_file; + std::filesystem::path readme_file; + }; + + struct ExportStats + { + size_t scans = 0; + size_t trajectory_nodes = 0; + uint64_t odometry_points = 0; // near_added = 0 + uint64_t near_points = 0; // near_added = 1 + uint64_t check_band_points = 0; // near_added = 2 + uint64_t raw_points_read = 0; + uint64_t raw_no_return = 0; // (0,0,0) or gps_time 0: never exported + uint64_t raw_near_outside_trajectory = 0; // near candidates with no trajectory node within 10 ms + double range_min_m = 0.0; + double range_max_m = 0.0; + //! range_m histogram, 0.1 m bins from 0 m; the last bin collects everything beyond. + std::array range_histogram{}; + uint64_t laz_bytes = 0; + uint64_t trajectory_bytes = 0; + double filter_threshold_xy_inner = 0.0; + double filter_threshold_xy_outer = 0.0; + double threshould_output_filter = 0.0; + }; + + //! Called with the fraction done (0..1) and a short description of the current step. + using ExportProgress = std::function; + + //! Writes the Easy GUI results of one step 1 session: + //! - laz_file: LAS 1.4 point format 6 LAZ with all step 1 points plus the near points (see README.txt), in + //! step 1's coordinate frame, poses as step 2 applies them (session_poses.mrp x the scan's points); + //! - trajectory_file: CSV, one row per trajectory node; + //! - readme_file: plain-text description of both files. + //! Streams one scan (step 1 chunk) or one raw file at a time, so memory does not grow with the recording. + //! Returns false with a plain-language `error` on failure, or when `cancel` became true. + bool exportSession( + const ExportRequest& request, + const std::atomic& cancel, + const ExportProgress& progress, + ExportStats& stats, + std::string& error); + + //! The step 1 parameter file in a session folder (HDMapping_params_*.toml, written by save_result), or empty. + std::filesystem::path findSessionParameterFile(const std::filesystem::path& session_folder); +} // namespace easy_gui diff --git a/apps/easy_gui/src/las14_writer.cpp b/apps/easy_gui/src/las14_writer.cpp new file mode 100644 index 00000000..264a51b4 --- /dev/null +++ b/apps/easy_gui/src/las14_writer.cpp @@ -0,0 +1,208 @@ +#include "las14_writer.h" + +#include + +#include +#include +#include + +namespace easy_gui +{ + namespace + { + //! LAS 1.4 R15 "Extra Bytes" descriptor. Written by hand rather than with laszip_add_attribute(), which always + //! sets the "scale" option bit: readers then scale the values, so laspy would return the uint8 near_added flag as + //! float64. +#pragma pack(push, 1) + struct ExtraBytesDescriptor + { + uint8_t reserved[2]; + uint8_t data_type; + uint8_t options; // 0: no no_data/min/max/scale/offset + char name[32]; + uint8_t unused[4]; + uint8_t no_data[24]; + uint8_t min[24]; + uint8_t max[24]; + double scale[3]; + double offset[3]; + char description[32]; + }; +#pragma pack(pop) + static_assert(sizeof(ExtraBytesDescriptor) == 192); + + size_t attributeSize(ExtraBytesAttribute::Type type) + { + return type == ExtraBytesAttribute::Type::U8 ? 1 : 4; + } + + std::string laszipError(void* writer, const std::string& what) + { + laszip_CHAR* message = nullptr; + if (writer && !laszip_get_error(writer, &message) && message && *message) + return what + ": " + message; + return what; + } + + void copyField(char* destination, size_t size, const std::string& text) + { + std::memset(destination, 0, size); + std::memcpy(destination, text.data(), std::min(size, text.size())); + } + + constexpr uint16_t kPointFormat6RecordLength = 30; + } // namespace + + Las14Writer::~Las14Writer() + { + std::string ignored; + close(ignored); + } + + bool Las14Writer::open( + const std::string& file_name, + const std::vector& attributes, + const Eigen::Vector3d& offset, + double scale, + const std::string& generating_software, + std::string& error) + { + if (laszip_create(&writer_)) + { + writer_ = nullptr; + error = "could not create the LAZ writer"; + return false; + } + + laszip_header* header = nullptr; + if (laszip_get_header_pointer(writer_, &header)) + { + error = laszipError(writer_, "could not access the LAZ header"); + return false; + } + + extra_bytes_size_ = 0; + for (const auto& a : attributes) + extra_bytes_size_ += attributeSize(a.type); + + header->file_source_ID = 0; + header->global_encoding = (1 << 4); // bit 0 = 0: not adjusted GPS time; bit 4: WKT (required for PF6) + header->version_major = 1; + header->version_minor = 4; + copyField(header->system_identifier, sizeof(header->system_identifier), "HDMapping Easy GUI"); + copyField(header->generating_software, sizeof(header->generating_software), generating_software); + { + const std::time_t now = std::time(nullptr); + std::tm tm{}; +#ifdef _WIN32 + gmtime_s(&tm, &now); +#else + gmtime_r(&now, &tm); +#endif + header->file_creation_day = static_cast(tm.tm_yday + 1); + header->file_creation_year = static_cast(tm.tm_year + 1900); + } + header->header_size = 375; + header->offset_to_point_data = 375; // laszip_add_vlr() below adds the VLR sizes + header->point_data_format = 6; + header->point_data_record_length = static_cast(kPointFormat6RecordLength + extra_bytes_size_); + header->x_scale_factor = header->y_scale_factor = header->z_scale_factor = scale; + header->x_offset = offset.x(); + header->y_offset = offset.y(); + header->z_offset = offset.z(); + + if (!attributes.empty()) + { + std::vector descriptors(attributes.size()); + for (size_t i = 0; i < attributes.size(); i++) + { + ExtraBytesDescriptor& d = descriptors[i]; + std::memset(&d, 0, sizeof(d)); + d.data_type = static_cast(attributes[i].type); + copyField(d.name, sizeof(d.name), attributes[i].name); + copyField(d.description, sizeof(d.description), attributes[i].description); + } + if (laszip_add_vlr( + writer_, + "LASF_Spec", + 4, + static_cast(descriptors.size() * sizeof(ExtraBytesDescriptor)), + "extra bytes", + reinterpret_cast(descriptors.data()))) + { + error = laszipError(writer_, "could not add the extra bytes record"); + return false; + } + } + + laszip_preserve_generating_software(writer_, 1); + const laszip_BOOL compress = file_name.size() >= 4 && file_name.compare(file_name.size() - 4, 4, ".laz") == 0; + if (laszip_open_writer(writer_, file_name.c_str(), compress)) + { + error = laszipError(writer_, "could not open '" + file_name + "' for writing"); + return false; + } + + laszip_point* point = nullptr; + if (laszip_get_point_pointer(writer_, &point)) + { + error = laszipError(writer_, "could not access the LAZ point"); + return false; + } + if (static_cast(point->num_extra_bytes) != extra_bytes_size_) + { + error = "LAZ writer has " + std::to_string(point->num_extra_bytes) + " extra bytes per point, expected " + + std::to_string(extra_bytes_size_); + return false; + } + point->extended_point_type = 1; + point->return_number = 1; + point->number_of_returns = 1; + point->extended_return_number = 1; + point->extended_number_of_returns = 1; + point_ = point; + points_written_ = 0; + return true; + } + + bool Las14Writer::write(const Las14Point& p, std::string& error) + { + auto* point = static_cast(point_); + point->gps_time = p.gps_time; + point->intensity = p.intensity; + point->point_source_ID = p.point_source_id; + point->user_data = p.user_data; + if (extra_bytes_size_ > 0) + { + if (p.extra_bytes) + std::memcpy(point->extra_bytes, p.extra_bytes, extra_bytes_size_); + else + std::memset(point->extra_bytes, 0, extra_bytes_size_); + } + + laszip_F64 coordinates[3] = { p.position.x(), p.position.y(), p.position.z() }; + if (laszip_set_coordinates(writer_, coordinates) || laszip_write_point(writer_) || laszip_update_inventory(writer_)) + { + error = laszipError(writer_, "writing point " + std::to_string(points_written_)); + return false; + } + points_written_++; + return true; + } + + bool Las14Writer::close(std::string& error) + { + if (!writer_) + return true; + bool ok = true; + if (point_ && laszip_close_writer(writer_)) + { + error = laszipError(writer_, "could not finish the LAZ file"); + ok = false; + } + laszip_destroy(writer_); + writer_ = nullptr; + point_ = nullptr; + return ok; + } +} // namespace easy_gui diff --git a/apps/easy_gui/src/las14_writer.h b/apps/easy_gui/src/las14_writer.h new file mode 100644 index 00000000..a1af6ada --- /dev/null +++ b/apps/easy_gui/src/las14_writer.h @@ -0,0 +1,73 @@ +#pragma once + +#include + +#include +#include +#include + +namespace easy_gui +{ + //! One named extra-bytes attribute (LAS 1.4 "Extra Bytes" VLR, record id 4). + struct ExtraBytesAttribute + { + enum class Type : uint8_t + { + U8 = 1, // LAS 1.4 data_type codes + F32 = 9 + }; + Type type; + std::string name; // max 32 characters + std::string description; // max 32 characters + }; + + struct Las14Point + { + Eigen::Vector3d position; // in the file's coordinate frame, before header offset/scale + double gps_time = 0.0; + uint16_t intensity = 0; + uint16_t point_source_id = 0; + uint8_t user_data = 0; + const uint8_t* extra_bytes = nullptr; // packed attribute values in declaration order, or null for all zeros + }; + + //! Streaming LAZ writer for LAS 1.4, point format 6, with extra-bytes attributes. Points are written one by one; + //! header counts and bounds are filled in at close() (laszip inventory), so nothing is buffered here. + //! + //! Global encoding: bit 0 (GPS time type) = 0, because gps_time holds Unix epoch seconds, not adjusted standard + //! GPS time; bit 4 (WKT) = 1, which LAS 1.4 requires for point formats 6-10 (no CRS record is written). + class Las14Writer + { + public: + Las14Writer() = default; + ~Las14Writer(); + Las14Writer(const Las14Writer&) = delete; + Las14Writer& operator=(const Las14Writer&) = delete; + + //! `scale` is the coordinate resolution in metres; `offset` the header offset. + bool open( + const std::string& file_name, + const std::vector& attributes, + const Eigen::Vector3d& offset, + double scale, + const std::string& generating_software, + std::string& error); + bool write(const Las14Point& point, std::string& error); + bool close(std::string& error); + + uint64_t pointsWritten() const + { + return points_written_; + } + size_t extraBytesSize() const + { + return extra_bytes_size_; + } + + private: + void* writer_ = nullptr; // laszip_POINTER + void* point_ = nullptr; // laszip_point* + size_t extra_bytes_size_ = 0; + uint64_t points_written_ = 0; + }; +} // namespace easy_gui diff --git a/apps/easy_gui/src/pipeline.cpp b/apps/easy_gui/src/pipeline.cpp index 77149901..ada37e7c 100644 --- a/apps/easy_gui/src/pipeline.cpp +++ b/apps/easy_gui/src/pipeline.cpp @@ -5,6 +5,7 @@ #include +#include #include #include #include @@ -45,6 +46,9 @@ namespace easy_gui , staging(root_ / "work" / "staging") , step1_parameters(root_ / "work" / "easy_gui_step1.toml") , session(root_ / "work" / "lio_session") + , point_cloud(root_ / "point_cloud.laz") + , trajectory(root_ / "trajectory.csv") + , readme(root_ / "README.txt") { } @@ -310,9 +314,79 @@ namespace easy_gui std::lock_guard lock(mutex_); status_.session_folder = out.session; } - cleanupWork(true); - finish(PipelineState::Succeeded, "Lidar odometry finished. The session is in '" + out.session.string() + "'."); + log("[easy_gui] " + timestamp() + " lidar odometry finished, exporting"); + runExport(check, out, cleanupWork); } busy_ = false; } + + void Pipeline::runExport(const PreflightResult& check, const OutputLayout& out, const std::function& cleanupWork) + { + setStage(PipelineStage::Export); + ExportRequest request; + request.session_file = out.session / "session.mjs"; + request.recording = check.files; + request.lidar_id = std::max(0, check.lidar_id); + request.near_min_range_m = request_.settings.near_min_range_m; + request.check_band_outer_m = request_.check_band_outer_m; + request.laz_file = out.point_cloud; + request.trajectory_file = out.trajectory; + request.readme_file = out.readme; + + std::string last_phase; + auto progress = [&](double fraction, const std::string& step) + { + // One log line per phase, not per scan/file: "Writing the point cloud (scan 3 of 113)" -> "Writing the point cloud" + const std::string phase = step.substr(0, step.find(" (")); + if (phase != last_phase) + log("[easy_gui] " + timestamp() + " export: " + phase); + last_phase = phase; + std::lock_guard lock(mutex_); + status_.export_fraction = fraction; + status_.export_step = step; + }; + + ExportStats stats; + std::string error; + const bool ok = exportSession(request, cancel_requested_, progress, stats, error); + std::error_code ec; + if (!ok) + { + for (const auto& f : { out.point_cloud, out.trajectory, out.readme }) + fs::remove(f, ec); + if (cancel_requested_) + { + cleanupWork(false); + finish(PipelineState::Cancelled, "Processing was cancelled. Nothing was kept except the log file."); + return; + } + // NOTE: unlike the other failures, the odometry session stays: it took most of the run, and step 2 can + // open and export it. + cleanupWork(true); + finish( + PipelineState::Failed, + "Lidar odometry finished, but the export failed: " + error + " The lidar odometry result was kept in '" + + out.session.string() + "'."); + return; + } + + { + std::ostringstream s; + s << "[easy_gui] export: " << stats.odometry_points << " odometry points + " << stats.near_points << " near points"; + if (stats.check_band_points) + s << " + " << stats.check_band_points << " check-band points"; + s << " (" << stats.raw_no_return << " of " << stats.raw_points_read << " raw points were no-return, " + << stats.raw_near_outside_trajectory << " near candidates outside the trajectory), range " << stats.range_min_m << " - " + << stats.range_max_m << " m, " << stats.trajectory_nodes << " trajectory poses; point cloud " << stats.laz_bytes + << " bytes, trajectory " << stats.trajectory_bytes << " bytes"; + log(s.str()); + } + { + std::lock_guard lock(mutex_); + status_.export_fraction = 1.0; + status_.export_stats = stats; + } + cleanupWork(true); + finish(PipelineState::Succeeded, "Done. The point cloud and trajectory are in '" + out.root.string() + "'."); + } } // namespace easy_gui diff --git a/apps/easy_gui/src/pipeline.h b/apps/easy_gui/src/pipeline.h index e3368350..bd38f981 100644 --- a/apps/easy_gui/src/pipeline.h +++ b/apps/easy_gui/src/pipeline.h @@ -1,6 +1,7 @@ #pragma once #include "child_process.h" +#include "export.h" #include "preflight.h" #include "progress_parser.h" #include "settings.h" @@ -10,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -23,6 +25,8 @@ namespace easy_gui std::filesystem::path output_folder; std::filesystem::path step1_executable; Settings settings; + //! Dev/test only (ExportRequest::check_band_outer_m): > 0 adds alignment-check points to the export. + double check_band_outer_m = 0.0; }; enum class PipelineState @@ -39,6 +43,7 @@ namespace easy_gui Checking, Staging, Odometry, + Export, Cleanup }; @@ -51,6 +56,9 @@ namespace easy_gui std::string detail; // technical detail for the "details" view double elapsed_s = 0.0; std::filesystem::path session_folder; // step 1 session (session.mjs) once odometry succeeded + double export_fraction = 0.0; // 0..1 during PipelineStage::Export + std::string export_step; // what the export is doing right now + ExportStats export_stats; // filled once the export succeeded }; //! Layout of an Easy GUI output folder. @@ -63,13 +71,17 @@ namespace easy_gui std::filesystem::path staging; // work/staging: links to the recording; step 1 puts cache/ and preview/ here std::filesystem::path step1_parameters; // work/easy_gui_step1.toml std::filesystem::path session; // work/lio_session: step 1 output (session.mjs, scan_lio_N.laz, ...) + std::filesystem::path point_cloud; // point_cloud.laz: the exported point cloud + std::filesystem::path trajectory; // trajectory.csv + std::filesystem::path readme; // README.txt: describes the two files above }; //! Locates lidar_odometry_step_1 next to the Easy GUI executable (plain build name or the installed, //! HDMAPPING_BINARY_PREFIX-ed one). The HDMAPPING_STEP1_EXECUTABLE environment variable overrides the search. std::filesystem::path findStep1Executable(); - //! Runs the whole Easy GUI processing on a worker thread: re-check, stage, step 1 in a child process, clean up. + //! Runs the whole Easy GUI processing on a worker thread: re-check, stage, step 1 in a child process, export, clean + //! up. //! The GUI polls status() every frame; cancel() kills the child process. class Pipeline { @@ -91,6 +103,7 @@ namespace easy_gui private: void run(); + void runExport(const PreflightResult& check, const OutputLayout& out, const std::function& cleanupWork); void log(const std::string& line); void setStage(PipelineStage stage); void finish(PipelineState state, const std::string& message, const std::string& detail = ""); diff --git a/apps/easy_gui/src/preflight.cpp b/apps/easy_gui/src/preflight.cpp index a47d923d..1ea70974 100644 --- a/apps/easy_gui/src/preflight.cpp +++ b/apps/easy_gui/src/preflight.cpp @@ -243,8 +243,9 @@ namespace easy_gui uint64_t estimateResultBytes(uint64_t raw_laz_bytes) { - // session ~0.85x + export ~1.5x + CSV, rounded up to 2.5x the raw LAZ size. - return raw_laz_bytes * 5 / 2; + // Measured on a 117 M-point forest walk (734 MB raw LAZ): session 0.82x + point cloud 1.24x + trajectory CSV + // 0.03x = 2.08x; 2.25x leaves some margin (the near-point share, and so the LAZ size, depends on the site). + return raw_laz_bytes * 9 / 4; } RecordingFiles classifyRecordingFiles(const fs::path& recording) diff --git a/apps/easy_gui/src/preflight.h b/apps/easy_gui/src/preflight.h index 18624cc3..03d3e979 100644 --- a/apps/easy_gui/src/preflight.h +++ b/apps/easy_gui/src/preflight.h @@ -66,9 +66,8 @@ namespace easy_gui //! some margin. uint64_t estimateWorkBytes(uint64_t total_points); - //! Disk space of what is kept: the step 1 session (scan_lio LAZ, ~0.85x the raw LAZ size measured) plus the - //! exported LAZ (1.4 PF6 with extra bytes and the near points, ~1.5x raw assumed until M2 measures it) and the - //! trajectory CSV (small). + //! Disk space of what is kept: the step 1 session (scan_lio LAZ, 0.82x the raw LAZ size measured), the exported + //! LAZ (1.4 PF6 with extra bytes and the near points, 1.24x measured) and the trajectory CSV (0.03x): 2.25x. uint64_t estimateResultBytes(uint64_t raw_laz_bytes); //! True if `inner` is `outer` or lies inside it (both made absolute and normalised first). diff --git a/apps/easy_gui/tests/CMakeLists.txt b/apps/easy_gui/tests/CMakeLists.txt index 5c2085b6..ed521715 100644 --- a/apps/easy_gui/tests/CMakeLists.txt +++ b/apps/easy_gui/tests/CMakeLists.txt @@ -4,11 +4,13 @@ project(easy_gui_tests) # Unit tests for apps/easy_gui's easy_gui_core (everything but the window). # Doctest, like the other suites. The pre-flight and staging tests write small -# synthetic Mandeye recordings to the temp directory; the progress parser test +# synthetic Mandeye recordings to the temp directory (the export tests also a +# synthetic step 1 session in step 1's file formats); the progress parser test # replays an excerpt of a real lidar_odometry_step_1 log. add_executable(easy_gui_tests test_main.cpp test_child_process.cpp + test_export.cpp test_pipeline_integration.cpp test_preflight.cpp test_progress_parser.cpp diff --git a/apps/easy_gui/tests/test_export.cpp b/apps/easy_gui/tests/test_export.cpp new file mode 100644 index 00000000..11ef8418 --- /dev/null +++ b/apps/easy_gui/tests/test_export.cpp @@ -0,0 +1,498 @@ +#include + +#include "export.h" +#include "forestry_preset.h" +#include "las14_writer.h" +#include "test_helpers.h" + +#include + +#include +#include + +#include +#include +#include +#include +#include + +using namespace easy_gui; +namespace fs = std::filesystem; + +namespace +{ + struct ReadPoint + { + Eigen::Vector3d xyz; + double gps_time; + uint16_t intensity; + uint16_t point_source_id; + uint8_t user_data; + float range_m; + uint8_t near_added; + }; + + struct ReadFile + { + laszip_header header{}; + std::vector extra_names; + std::vector extra_types; + std::vector extra_options; + std::vector points; + }; + + ReadFile readLaz(const fs::path& file) + { + ReadFile r; + laszip_POINTER reader = nullptr; + REQUIRE(laszip_create(&reader) == 0); + laszip_BOOL compressed = 0; + REQUIRE(laszip_open_reader(reader, file.string().c_str(), &compressed) == 0); + laszip_header* h = nullptr; + laszip_get_header_pointer(reader, &h); + r.header = *h; + for (laszip_U32 v = 0; v < h->number_of_variable_length_records; v++) + { + const auto& vlr = h->vlrs[v]; + if (std::string(vlr.user_id) == "LASF_Spec" && vlr.record_id == 4) + for (size_t k = 0; k + 192 <= vlr.record_length_after_header; k += 192) + { + const laszip_U8* d = vlr.data + k; + r.extra_types.push_back(d[2]); + r.extra_options.push_back(d[3]); + r.extra_names.emplace_back(reinterpret_cast(d + 4), strnlen(reinterpret_cast(d + 4), 32)); + } + } + laszip_point* p = nullptr; + laszip_get_point_pointer(reader, &p); + const uint64_t n = h->number_of_point_records ? h->number_of_point_records : h->extended_number_of_point_records; + for (uint64_t i = 0; i < n; i++) + { + REQUIRE(laszip_read_point(reader) == 0); + ReadPoint q{}; + q.xyz = { h->x_offset + h->x_scale_factor * p->X, + h->y_offset + h->y_scale_factor * p->Y, + h->z_offset + h->z_scale_factor * p->Z }; + q.gps_time = p->gps_time; + q.intensity = p->intensity; + q.point_source_id = p->point_source_ID; + q.user_data = p->user_data; + if (p->num_extra_bytes >= 5) + { + std::memcpy(&q.range_m, p->extra_bytes, 4); + q.near_added = p->extra_bytes[4]; + } + r.points.push_back(q); + } + laszip_close_reader(reader); + laszip_destroy(reader); + return r; + } + + struct RawPoint + { + Eigen::Vector3d xyz; + double t; + uint16_t intensity = 7; + }; + + //! Mandeye-style raw LAZ (LAS 1.2 PF1, lidar frame) with the given points. + void writeRaw(const fs::path& file, const std::vector& points, int lidar_id) + { + laszip_POINTER writer = nullptr; + REQUIRE(laszip_create(&writer) == 0); + laszip_header* header = nullptr; + laszip_get_header_pointer(writer, &header); + header->version_major = 1; + header->version_minor = 2; + header->point_data_format = 1; + header->point_data_record_length = 28; + header->x_scale_factor = header->y_scale_factor = header->z_scale_factor = 0.0001; + REQUIRE(laszip_open_writer(writer, file.string().c_str(), 1) == 0); + laszip_point* point = nullptr; + laszip_get_point_pointer(writer, &point); + for (const auto& p : points) + { + point->gps_time = p.t; + point->user_data = static_cast(lidar_id); + point->intensity = p.intensity; + laszip_F64 xyz[3] = { p.xyz.x(), p.xyz.y(), p.xyz.z() }; + laszip_set_coordinates(writer, xyz); + laszip_write_point(writer); + laszip_update_inventory(writer); + } + laszip_close_writer(writer); + laszip_destroy(writer); + } + + void writePoseFile(const fs::path& file, const std::vector& poses, const std::vector& names) + { + std::ofstream out(file); + out << poses.size() << "\n" << std::setprecision(17); + for (size_t i = 0; i < poses.size(); i++) + { + out << names[i] << "\n"; + const auto& m = poses[i].matrix(); + for (int r = 0; r < 4; r++) + out << m(r, 0) << " " << m(r, 1) << " " << m(r, 2) << " " << m(r, 3) << "\n"; + } + } + + //! A small synthetic step 1 result: kScans scans of kNodes trajectory nodes at 200 Hz along a curved path, written + //! in exactly step 1's file formats (save_result in lidar_odometry.cpp), plus the matching raw recording. + struct SyntheticSession + { + static constexpr int kScans = 2; + static constexpr int kNodes = 40; + static constexpr double kT0 = 1783328634.0; + static constexpr int kLidarId = 3; + + fs::path session_file; + RecordingFiles recording; + std::vector sensor_points; // per scan_lio point, in the sensor frame (for range_m) + size_t odometry_points = 0; + + static double nodeTime(int k) + { + return kT0 + 0.005 * k; + } + //! Global pose of node k (k counts across scans). + static Eigen::Affine3d pose(int k) + { + Eigen::Affine3d m = Eigen::Affine3d::Identity(); + m.linear() = (Eigen::AngleAxisd(0.5 + 0.02 * k, Eigen::Vector3d::UnitZ()) * Eigen::AngleAxisd(0.1, Eigen::Vector3d::UnitX())) + .toRotationMatrix(); + m.translation() = Eigen::Vector3d(10.0 + 0.005 * k, 5.0 + 0.002 * k, 1.0); + return m; + } + + explicit SyntheticSession(const fs::path& root) + { + const fs::path session = root / "lio_session"; + const fs::path raw = root / "recording"; + fs::create_directories(session); + fs::create_directories(raw); + + std::vector scan_poses; + std::vector names; + nlohmann::json laz_names; + for (int s = 0; s < kScans; s++) + { + const int first = s * kNodes; + const Eigen::Affine3d m_pose = pose(first); + scan_poses.push_back(m_pose); + names.push_back("scan_lio_" + std::to_string(s) + ".laz"); + laz_names.push_back({ { "file_name", (session / names.back()).string() } }); + + std::ofstream trj(session / ("trajectory_lio_" + std::to_string(s) + ".csv")); + trj << "timestamp_nanoseconds pose00 pose01 pose02 pose03 pose10 pose11 pose12 pose13 pose20 pose21 pose22 pose23 " + "timestampUnix_nanoseconds om_rad fi_rad ka_rad\n"; + std::vector local_points; + std::vector intensity; + std::vector timestamps; + for (int j = 0; j < kNodes; j++) + { + const Eigen::Affine3d local = m_pose.inverse() * pose(first + j); + trj << std::setprecision(20) << nodeTime(first + j) * 1e9 << " " << std::setprecision(10); + for (int r = 0; r < 3; r++) + for (int c = 0; c < 4; c++) + trj << local(r, c) << " "; + trj << std::setprecision(20) << std::floor(nodeTime(first + j) * 1000) << " 0.01 0.02 0.03 \n"; + + // two odometry points per node, 1 ms after it: one near, one far + for (const Eigen::Vector3d p_sensor : { Eigen::Vector3d(1.2, 0.3, 0.4), Eigen::Vector3d(-8.0, 3.0, 2.0) }) + { + local_points.push_back(local * p_sensor); + intensity.push_back(static_cast(20 + j)); + timestamps.push_back(nodeTime(first + j) + 0.001); // exportLaz writes seconds x 1e9, as step 1 + sensor_points.push_back(p_sensor); + } + } + REQUIRE(exportLaz((session / names.back()).string(), local_points, intensity, timestamps)); + odometry_points += local_points.size(); + } + writePoseFile(session / "session_poses.mrp", scan_poses, names); + writePoseFile(session / "session_ini_poses.mri", scan_poses, names); + std::string error; + REQUIRE(writeStep1ParameterFile(session / "HDMapping_params_test.toml", error)); + + nlohmann::json settings = { { "offset_x", 0.0 }, + { "offset_y", 0.0 }, + { "offset_z", 0.0 }, + { "folder_name", session.string() }, + { "out_folder_name", session.string() }, + { "poses_file_name", (session / "session_poses.mrp").string() }, + { "initial_poses_file_name", (session / "session_ini_poses.mri").string() }, + { "out_poses_file_name", (session / "session_poses.mrp").string() } }; + session_file = session / "session.mjs"; + std::ofstream(session_file) << nlohmann::json{ { "Session Settings", settings }, { "laz_file_names", laz_names } }.dump(2); + + recording.laz_files = { raw / "lidar0000.laz", raw / "lidar0001.laz" }; + recording.csv_files = { raw / "imu0000.csv", raw / "imu0001.csv" }; + recording.sn_file = raw / "lidar0000.sn"; + recording.pairs = 2; + easy_gui_test::writeText(recording.sn_file, std::to_string(kLidarId) + " ARMCP1U0038132\n"); + } + }; + + bool near(const Eigen::Vector3d& a, const Eigen::Vector3d& b, double tolerance = 0.0002) + { + return (a - b).norm() <= tolerance; + } +} // namespace + +TEST_CASE("Las14Writer writes LAS 1.4 point format 6 with named, unscaled extra bytes") +{ + easy_gui_test::TempDir tmp; + const fs::path file = tmp.path / "out.laz"; + Las14Writer writer; + std::string error; + REQUIRE(writer.open( + file.string(), + { { ExtraBytesAttribute::Type::F32, "range_m", "range" }, { ExtraBytesAttribute::Type::U8, "near_added", "flag" } }, + Eigen::Vector3d(100.0, 200.0, 0.0), + 0.0001, + "test", + error)); +#pragma pack(push, 1) + struct + { + float r; + uint8_t f; + } extra{ 12.5f, 1 }; +#pragma pack(pop) + Las14Point p; + p.position = { 101.23456, 198.5, -3.25 }; + p.gps_time = 1783328634.123456; + p.intensity = 77; + p.point_source_id = 4321; + p.user_data = 2; + p.extra_bytes = reinterpret_cast(&extra); + REQUIRE(writer.write(p, error)); + p.extra_bytes = nullptr; // all-zero extra bytes + p.position = { 99.0, 201.0, 5.0 }; + REQUIRE(writer.write(p, error)); + REQUIRE(writer.close(error)); + + const ReadFile r = readLaz(file); + CHECK(r.header.version_major == 1); + CHECK(r.header.version_minor == 4); + CHECK(r.header.point_data_format == 6); + CHECK(r.header.point_data_record_length == 35); + CHECK((r.header.global_encoding & 1) == 0); // gps_time is not adjusted GPS time + CHECK((r.header.global_encoding & 16) == 16); // WKT bit, required for PF6 + CHECK(r.header.extended_number_of_point_records == 2); + CHECK(r.header.number_of_point_records == 0); // legacy counter stays 0 for PF6 + CHECK(r.header.min_x == doctest::Approx(99.0)); + CHECK(r.header.max_z == doctest::Approx(5.0)); + REQUIRE(r.extra_names.size() == 2); + CHECK(r.extra_names[0] == "range_m"); + CHECK(r.extra_names[1] == "near_added"); + CHECK(r.extra_types[0] == 9); // float + CHECK(r.extra_types[1] == 1); // unsigned char + CHECK(r.extra_options[0] == 0); // no scale: readers return the raw float / uint8 + CHECK(r.extra_options[1] == 0); + REQUIRE(r.points.size() == 2); + CHECK(near(r.points[0].xyz, { 101.2346, 198.5, -3.25 }, 1e-6)); + CHECK(r.points[0].gps_time == 1783328634.123456); + CHECK(r.points[0].intensity == 77); + CHECK(r.points[0].point_source_id == 4321); + CHECK(r.points[0].user_data == 2); + CHECK(r.points[0].range_m == 12.5f); + CHECK(r.points[0].near_added == 1); + CHECK(r.points[1].range_m == 0.0f); + CHECK(r.points[1].near_added == 0); +} + +TEST_CASE("exportSession: step 1 points as step 2 places them, near points with step 1's transform chain") +{ + easy_gui_test::TempDir tmp; + SyntheticSession s(tmp.path); + using S = SyntheticSession; + + // Raw points. Node 5 lies in scan 0, node 47 in scan 1 (7th node of that scan). + const double t5 = S::nodeTime(5) + 0.002; + const double t47 = S::nodeTime(47) + 0.004; + const Eigen::Vector3d near_a(0.5, 0.3, 0.2); // horizontal 0.58 m, 3D 0.62 m: near point + const Eigen::Vector3d near_steep(0.6, 0.0, 1.5); // horizontal 0.6 m but 3D 1.62 m: still a near point + const Eigen::Vector3d band(1.1, 0.0, 0.0); // step 1 exports this one itself + std::vector file0 = { + { near_a, t5, 11 }, + { Eigen::Vector3d(0.2, 0.1, 0.0), t5 }, // 3D 0.22 m < near_min_range + { Eigen::Vector3d::Zero(), t5 }, // no return + { Eigen::Vector3d(0.5, 0.0, 0.0), 0.0 }, // time 0 + { Eigen::Vector3d(3.0, 0.0, 0.0), t5 }, // step 1's point + { band, t5 }, + { near_a, S::kT0 - 1.0 }, // before the trajectory: no pose within 10 ms + }; + std::vector file1 = { + { near_steep, t47, 12 }, + { near_a, S::nodeTime(2 * S::kNodes - 1) + 0.02 }, // 20 ms after the last node + }; + writeRaw(s.recording.laz_files[0], file0, S::kLidarId); + writeRaw(s.recording.laz_files[1], file1, S::kLidarId); + + ExportRequest request; + request.session_file = s.session_file; + request.recording = s.recording; + request.lidar_id = S::kLidarId; + request.near_min_range_m = 0.4; + request.laz_file = tmp.path / "out" / "point_cloud.laz"; + request.trajectory_file = tmp.path / "out" / "trajectory.csv"; + request.readme_file = tmp.path / "out" / "README.txt"; + + std::atomic cancel{ false }; + double last_fraction = -1.0; + bool monotonic = true; + ExportStats stats; + std::string error; + REQUIRE_MESSAGE( + exportSession( + request, + cancel, + [&](double f, const std::string&) + { + monotonic = monotonic && f >= last_fraction; + last_fraction = f; + }, + stats, + error), + error); + CHECK(monotonic); + CHECK(last_fraction == 1.0); + + CHECK(stats.scans == 2); + CHECK(stats.trajectory_nodes == 2 * S::kNodes); + CHECK(stats.odometry_points == s.odometry_points); + CHECK(stats.near_points == 2); + CHECK(stats.check_band_points == 0); + CHECK(stats.raw_points_read == file0.size() + file1.size()); + CHECK(stats.raw_no_return == 2); + CHECK(stats.raw_near_outside_trajectory == 2); + CHECK(stats.filter_threshold_xy_inner == 1.0); + CHECK(stats.threshould_output_filter == 1.0); // Easy GUI's output setting, read back from the session + + const ReadFile r = readLaz(request.laz_file); + REQUIRE(r.points.size() == s.odometry_points + 2); + + // odometry points: scan pose x local point, range from the sensor frame, ramp time in seconds + for (size_t i = 0; i < s.odometry_points; i++) + { + const auto& p = r.points[i]; + const int scan = static_cast(i / (2 * S::kNodes)); + const int node = scan * S::kNodes + static_cast(i % (2 * S::kNodes)) / 2; + CHECK(p.near_added == kFromOdometry); + CHECK(p.point_source_id == scan); + CHECK(p.user_data == S::kLidarId); + CHECK(near(p.xyz, S::pose(node) * s.sensor_points[i])); + CHECK(p.range_m == doctest::Approx(s.sensor_points[i].norm()).epsilon(1e-4)); + CHECK(p.gps_time == doctest::Approx(S::nodeTime(node) + 0.001).epsilon(1e-15)); + } + + // near points: pose of the last node before the raw time, raw time and intensity kept + const auto& a = r.points[s.odometry_points]; + CHECK(a.near_added == kNearAdded); + CHECK(near(a.xyz, S::pose(5) * near_a)); + CHECK(a.gps_time == t5); + CHECK(a.intensity == 11); + CHECK(a.point_source_id == 0); + CHECK(a.range_m == doctest::Approx(near_a.norm())); + const auto& b = r.points[s.odometry_points + 1]; + CHECK(b.near_added == kNearAdded); + CHECK(near(b.xyz, S::pose(47) * near_steep)); + CHECK(b.point_source_id == 1); + CHECK(b.range_m == doctest::Approx(near_steep.norm())); + CHECK(stats.range_min_m == doctest::Approx(near_a.norm()).epsilon(1e-6)); + + // trajectory: one row per node, global pose, Unix seconds + std::ifstream csv(request.trajectory_file); + std::string header, row; + std::getline(csv, header); + CHECK(header.starts_with("timestamp_unix_s,x,y,z,qw,qx,qy,qz")); + int rows = 0; + std::vector row47; + while (std::getline(csv, row)) + { + if (rows == 47) + { + std::stringstream ss(row); + std::string v; + while (std::getline(ss, v, ',')) + row47.push_back(std::stod(v)); + } + rows++; + } + CHECK(rows == 2 * S::kNodes); + REQUIRE(row47.size() == 16); + CHECK(row47[0] == doctest::Approx(S::nodeTime(47)).epsilon(1e-15)); + CHECK(near({ row47[1], row47[2], row47[3] }, S::pose(47).translation())); + const Eigen::Quaterniond q(S::pose(47).rotation()); + CHECK(std::abs(std::abs(q.dot(Eigen::Quaterniond(row47[4], row47[5], row47[6], row47[7]))) - 1.0) < 1e-8); + CHECK(row47[11] == 1); // scan_index + + std::ifstream readme(request.readme_file); + const std::string text((std::istreambuf_iterator(readme)), std::istreambuf_iterator()); + CHECK(text.find("Unix epoch seconds") != std::string::npos); + CHECK(text.find("at least 0.40 m") != std::string::npos); + CHECK(text.find("near_added") != std::string::npos); + + SUBCASE("check-band test mode adds the 1.0-1.2 m raw points with flag 2") + { + request.check_band_outer_m = 1.2; + REQUIRE(exportSession( + request, + cancel, + [](double, const std::string&) + { + }, + stats, + error)); + CHECK(stats.near_points == 2); + CHECK(stats.check_band_points == 1); + const ReadFile rb = readLaz(request.laz_file); + REQUIRE(rb.points.size() == s.odometry_points + 3); + const auto& c = rb.points[s.odometry_points + 1]; // raw file order: near_a, band, near_steep + CHECK(c.near_added == kCheckBand); + CHECK(near(c.xyz, S::pose(5) * band)); + CHECK(c.range_m == doctest::Approx(band.norm())); + } + + SUBCASE("cancel stops the export") + { + cancel = true; + CHECK_FALSE(exportSession( + request, + cancel, + [](double, const std::string&) + { + }, + stats, + error)); + CHECK(error.find("cancelled") != std::string::npos); + } +} + +TEST_CASE("exportSession refuses a session without its parameter file, in plain language") +{ + easy_gui_test::TempDir tmp; + SyntheticSession s(tmp.path); + fs::remove(s.session_file.parent_path() / "HDMapping_params_test.toml"); + ExportRequest request; + request.session_file = s.session_file; + request.recording = s.recording; + request.laz_file = tmp.path / "point_cloud.laz"; + request.trajectory_file = tmp.path / "trajectory.csv"; + request.readme_file = tmp.path / "README.txt"; + std::atomic cancel{ false }; + ExportStats stats; + std::string error; + CHECK_FALSE(exportSession( + request, + cancel, + [](double, const std::string&) + { + }, + stats, + error)); + CHECK(error.find("settings file") != std::string::npos); +} diff --git a/apps/easy_gui/tests/test_pipeline_integration.cpp b/apps/easy_gui/tests/test_pipeline_integration.cpp index a079374e..f1db6315 100644 --- a/apps/easy_gui/tests/test_pipeline_integration.cpp +++ b/apps/easy_gui/tests/test_pipeline_integration.cpp @@ -98,3 +98,30 @@ TEST_CASE("a recording too short for odometry fails with a message, not a crash" CHECK_FALSE(s.message.empty()); MESSAGE("message shown to the user: " << s.message); } + +TEST_CASE("a real run ends with the point cloud, the trajectory and the README" * doctest::skip(testRecording() == nullptr)) +{ + easy_gui_test::TempDir tmp; + const fs::path out = tmp.path / "out"; + Pipeline pipeline; + std::string error; + REQUIRE(pipeline.start(PipelineRequest{ testRecording(), out, findStep1Executable(), Settings{} }, error)); + const PipelineStatus s = waitUntil( + pipeline, + [](const PipelineStatus& st) + { + return st.state != PipelineState::Running; + }, + std::chrono::seconds(600)); + REQUIRE_MESSAGE(s.state == PipelineState::Succeeded, s.message); + const OutputLayout layout(out); + CHECK(fs::file_size(layout.point_cloud) == s.export_stats.laz_bytes); + CHECK(fs::file_size(layout.trajectory) == s.export_stats.trajectory_bytes); + CHECK(fs::exists(layout.readme)); + CHECK(s.export_stats.odometry_points > 0); + CHECK(s.export_stats.near_points > 0); + CHECK(s.export_stats.check_band_points == 0); + CHECK(s.export_stats.range_min_m >= Settings{}.near_min_range_m); + CHECK(fs::exists(layout.session / "session.mjs")); // kept for step 2 + CHECK_FALSE(fs::exists(layout.staging)); +} diff --git a/apps/easy_gui/tests/test_preflight.cpp b/apps/easy_gui/tests/test_preflight.cpp index 79723855..cebf0c3e 100644 --- a/apps/easy_gui/tests/test_preflight.cpp +++ b/apps/easy_gui/tests/test_preflight.cpp @@ -47,11 +47,11 @@ TEST_CASE("a plausible recording passes with a warning about the unmatched file" CHECK(hasFinding(r, Severity::Ok, "Recording looks good")); CHECK(hasFinding(r, Severity::Warning, "lidar0002.laz")); - // Disk estimate: 120 B per raw point of work space + 2.5x the paired raw LAZ size of results, always reported. + // Disk estimate: 120 B per raw point of work space + 2.25x the paired raw LAZ size of results, always reported. const uint64_t laz_bytes = fs::file_size(tmp.path / "rec" / "lidar0000.laz") + fs::file_size(tmp.path / "rec" / "lidar0001.laz"); CHECK(r.raw_laz_bytes == laz_bytes); CHECK(r.estimated_work_bytes == 4000 * 120); - CHECK(r.estimated_result_bytes == laz_bytes * 5 / 2); + CHECK(r.estimated_result_bytes == laz_bytes * 9 / 4); CHECK(r.estimated_disk_bytes == r.estimated_work_bytes + r.estimated_result_bytes); CHECK(hasFinding(r, Severity::Ok, "Disk space: about")); } diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt index 5e38ab80..d6a90b0f 100644 --- a/core/CMakeLists.txt +++ b/core/CMakeLists.txt @@ -116,17 +116,17 @@ function(add_core_target target_name with_gui) endif() endfunction() -# core_no_gui is used by python bindings, lidar odometry test subprojects and session_to_mcap (console tools) only - if they are not build just do not waste time building core_no_gui -if(BUILD_TESTING OR BUILD_WITH_PYBIND OR BUILD_WITH_CLI_TOOLS) - add_core_target(core_no_gui FALSE) +# core_no_gui is used by python bindings, lidar odometry test subprojects, session_to_mcap (console tools) and +# apps/easy_gui. Easy GUI is always built (its export reads sessions with Session), so core_no_gui is too; it was +# previously only built with BUILD_TESTING, BUILD_WITH_PYBIND or BUILD_WITH_CLI_TOOLS. +add_core_target(core_no_gui FALSE) - target_precompile_headers(core_no_gui - PRIVATE include/pch/pch.h - ) +target_precompile_headers(core_no_gui + PRIVATE include/pch/pch.h +) - set_target_properties(core_no_gui PROPERTIES - POSITION_INDEPENDENT_CODE ON) -endif() +set_target_properties(core_no_gui PROPERTIES + POSITION_INDEPENDENT_CODE ON) add_core_target(core TRUE) target_link_libraries(core PUBLIC core_pfd) From 187aa19af1f695d8649fc6f659cd6c51cf141e5d Mon Sep 17 00:00:00 2001 From: Martin Mokros Date: Sat, 17 Oct 2026 20:54:12 +0100 Subject: [PATCH 03/23] Add Easy GUI place_by_raw_time export option (default off); add CLAUDE.md place_by_raw_time places every point from the raw files with the trajectory pose at its real gps_time instead of taking step 1's points (placed at step 1's even-spread ramp time), and writes the raw gps_time. Step 1's distance filters and the near-point rule are unchanged; the raw points step 1 drops between its 0.1 s pieces are included. Available as Settings / ExportRequest::place_by_raw_time, --place-by-raw-time and an experimental checkbox; the default stays off until its effect on sharpness is measured. On the 117 M-point test walk: 93,359,291 instead of 89,326,756 points inside step 1's limits (+4,026,808 from the inter-chunk gaps). For the common points, 79 % get the same pose; moved points shift by a median 8 / 13 / 23 / 47 mm at 1-3 / 3-5 / 5-10 / 10-20 m (a neighbouring pose's rotation, ~9 mrad). CLAUDE.md documents the repository and the Easy GUI decisions and measurements (M1, M2, this option). Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 144 ++++++++++++++++++++++++++++ apps/easy_gui/src/easy_gui_main.cpp | 48 +++++++--- apps/easy_gui/src/export.cpp | 71 +++++++++----- apps/easy_gui/src/export.h | 7 +- apps/easy_gui/src/pipeline.cpp | 5 +- apps/easy_gui/src/settings.h | 4 + apps/easy_gui/tests/test_export.cpp | 31 +++++- 7 files changed, 269 insertions(+), 41 deletions(-) create mode 100644 CLAUDE.md diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 00000000..7340d734 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,144 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Project + +HDMapping: C++20 desktop apps that turn raw Mandeye (handheld LiDAR + IMU) recordings into registered, georeferenced LAZ point clouds. Optimization is implemented from scratch with Eigen (Jacobians generated by `3rdparty/observation_equations`), with no Ceres, g2o or GTSAM. User-facing docs live in the GitHub wiki (https://github.com/MapsHD/HDMapping/wiki). + +## Build, test, format + +Dependencies are git submodules: `git submodule update --init --recursive` (or `python3 scripts/run_submodules_sync_update.py`). CMake >= 4.0 is required. raylib 5.5 and rlImGui are fetched at configure time via `FetchContent`, so the first configure needs network access. + +```bash +# Configure (same flags as CI; on macOS CI also passes -DFREEGLUT_COCOA=ON) +cmake -B build -DCMAKE_BUILD_TYPE=Release -DBUILD_WITH_UTILITY_APPLICATION=ON -DBUILD_TESTING=ON +cmake --build build --config Release -j + +# Build a single app +cmake --build build --target multi_view_tls_registration_step_2 + +# Run all tests (from the build dir) +cd build && ctest -C Release --output-on-failure +# Run one test suite: hdmapping_shared_tests, calib_core_tests, lidar_odometry_step_1_tests, hdmapping_rosbags_tests +ctest -R calib_core_tests --output-on-failure +# Run one doctest case directly +./build/bin/calib_core_tests --test-case="" + +# Format (CI enforces clang-format 21 via scripts/run_check_clang_format.py) +python3 scripts/run_clang_format.py +python3 scripts/run_check_clang_format.py +``` + +Binaries go to `build/bin`. Optional CMake switches: +- `BUILD_WITH_CLI_TOOLS`: `apps/console_tools` converters. +- `BUILD_WITH_HD_MAPPER_APPLICATION`: `hd_mapper`. +- `BUILD_WITH_UTILITY_APPLICATION`: calibration, forestry and Livox tools. +- `BUILD_WITH_PYBIND`: `pybind/` Python bindings. +- `HDMAPPING_PROFILER=NONE|UTL|TRACY`: profiler backend. +- `HD_CPU_OPTIMIZATION`: CPU optimization target. + +Tests use vendored doctest (`3rdparty/doctest`) and are only added when `BUILD_TESTING=ON`. A new test directory must also be registered in the root `CMakeLists.txt`. + +## Pipeline (see images/workflow.png) + +1. `mandeye_raw_data_viewer`: QC of the raw recording (`lidar*.laz`, `imu*.csv`, `*.sn`). +2. `lidar_odometry_step_1`: LiDAR-inertial odometry. `save_result()` in `apps/lidar_odometry_step_1/lidar_odometry.cpp` writes a session directory (by default the next free `/lio_result_N`): `session.mjs` (JSON), `scan_lio_N.laz`, `trajectory_lio_N.csv`, `session_poses.mrp` / `session_ini_poses.mri`, `point_sizes_per_chunk.json`, and the parameters TOML. Parameters are stored as TOML (`toml_io.cpp`). +3. `mandeye_single_session_viewer`: QC of the odometry result. +4. `multi_view_tls_registration_step_2`: refines one session (pose-graph loop closure, ICP/NDT, GNSS/GCP/control-point georeferencing) and exports the result. It also produces ground-truth sessions. +5. `multi_session_registration_step_3`: aligns several sessions using a project JSON (`session_file_names`) and `multi_session_factor_graph.cpp`, then writes poses back into each session. `multi_session_registration_no_gui.cpp` is the headless variant. + +The file that ties the steps together is the session JSON (`session.mjs` from step 1, `*.json` also accepted), loaded and saved by `Session` (`core/include/Core/session.h`). `Session::load` also accepts a single `.laz`/`.las` file (`load_from_point_cloud`). The CMake target names (`*_step_1/2/3`) differ from the app directory names. + +The step-1 sources (`lidar_odometry*.cpp`, `raw_cloud_loader.cpp`, `toml_io.cpp`, ...) are not a library. They are listed by relative path in the CMakeLists of the raw viewer, the single-session viewer and step 2. Changing their signatures affects all of those apps. + +## Libraries + +- `core/` builds several targets (see `core/CMakeLists.txt`): + - `core_math`: registration and optimization code, always `WITH_GUI=0`. Its generated Jacobian headers are slow to compile. + - `core` (`WITH_GUI=1`, adds the ImGui manual loop-closure and observation-picking panels) and `core_no_gui` (only built for tests, pybind and CLI tools). + - `core_pfd` (file dialogs), `core_e57` and `core_raylib` (shared raylib rendering). + - Classes shared between GUI and non-GUI builds (e.g. `Session`) must not change their layout under `WITH_GUI`. +- `calib_core/`: cameraโ€“LiDAR calibration library for the `camera_lidar_*` apps. +- `raylib_widgets/`: shared UI for the raylib apps. `AppShell` provides the dockspace, the Console dock and the info/shortcuts window; other parts are `OrbitCamera`, `PointPicking` and `PointBufferParts`. +- `shared/include/HDMapping/`: header-only helpers (`HDMAPPING_ASSERT`, pose interpolation). +- `rosbags/`: MCAP read/write. + +## GUI architecture (two stacks, mid-migration) + +All GUIs are immediate-mode Dear ImGui (vendored `3rdparty/imgui`, docking branch) with ImGuizmo and implot. + +- **Legacy: freeglut + OpenGL2.** Used by step 1, the raw and single-session viewers, the `*_legacy` apps and most utilities. These link the `imgui` target from `cmake/imgui.cmake`, which has the GLUT and OpenGL2 backends compiled in. GLUT callbacks drive `glutMainLoop()`, and the 3D view is fixed-function GL. +- **New: raylib + rlImGui.** Used by step 2, step 3, `session_uncertainty_calculation`, `session_to_session_rigid_registration` and the `camera_lidar_*` apps. + - `cmake/raylib.cmake` builds separate `imgui_raylib` and `imguizmo_raylib` targets from the same vendored sources, without any backend. Never link both `imgui` and `imgui_raylib` into one binary. + - Main loop pattern: `while (!WindowShouldClose()) { rlImGuiBegin(); ShowMainDockSpace(); ...; rlImGuiEnd(); }`, with drag and drop via `IsFileDropped()`. + - `IMGUI_DISABLE_OBSOLETE_FUNCTIONS` is deliberately not defined, because `core`'s GUI sources call obsolete ImGui APIs. +- The `*_legacy` directories are near-copies of the GLUT versions of steps 2 and 3, kept while the raylib ports mature. A fix to shared logic may need to be applied in both copies. + +## Conventions + +- clang-format (`.clang-format`) is checked in CI; run `scripts/run_clang_format.py` before committing. +- Comments in this codebase explain *why*, often with `NOTE(author)` / `TODO(author)` tags. CMake files carry detailed rationale comments for target splits; read them before restructuring targets. + +## Easy GUI (branch `easy-gui`) + +Goal: one app: raw Mandeye folder โ†’ step 1 (odometry, forestry preset) โ†’ step 2 refinement โ†’ export. No step 3. Outputs: (1) LAZ point cloud with all available per-point attributes, (2) trajectory CSV with timestamp, x, y, z and all other available fields. + +Constraints: raylib stack; reuse existing code, don't duplicate algorithms; don't modify `core_math`; existing apps stay unchanged; progress display and plain-language errors. + +Decisions: +- v1 = odometry โ†’ export, no refinement. Validate it against a manual step 1 โ†’ step 2 run first. v2 adds gated pose-graph loop closure (`PoseGraphLoopClosure`, auto-detected candidates, edges rejected if ICP fails) behind a toggle. Do not use `PoseGraphSLAM::optimize`. +- Output LAZ: LAS 1.4, point format 6, extra-bytes attributes. `gps_time` = Unix epoch seconds; global encoding bit 0 (GPS time type) = 0, since it is not adjusted GPS time. Document the time base. +- Multi-lidar rigs are out of scope for v1; the pre-flight check rejects recordings with more than one lidar id. +- Near points (added by Easy GUI at export, never seen by odometry): keep raw points with `near_min_range` โ‰ค 3D range **and** that step 1's distance filters removed. As implemented, that is the exact complement: horizontal range โ‰ค `filter_threshold_xy_inner` (1.0 m) or calibrated 3D norm โ‰ค `threshould_output_filter`, and horizontal range < `filter_threshold_xy_outer`. Always drop exact (0,0,0) and gps_time 0 no-return points (~15 % of raw points). Transform each one with step 1's rule at its *raw* gps_time: the last trajectory node with t_node < t, if within 10 ms (step 1's own tolerance), so `global = mrp[scan] ยท csv_pose[node] ยท calib ยท p_raw`. Mark it with an extra-bytes `near_added` flag (uint8; `user_data` keeps the lidar id). `near_min_range` is an advanced setting, default 0.4 m: measured raw data shows the operator's body/handle only as small blobs inside 0.4 m, and near points beyond that are environment in all directions. +- For the near rule to meet step 1's output without a gap, Easy GUI writes `threshould_output_filter = filter_threshold_xy_inner` (1.0 instead of the preset's 1.5). It is output-only, so the trajectory is unchanged; points with horizontal range โ‰ฅ 1.0 m then all come from step 1, and points below that come from the near rule. +- Easy GUI never writes into the recording folder (it may be on the Mandeye USB stick). `load_data` puts `cache/` and `preview/` next to the input files, so step 1 runs on a staging folder of symlinks inside the Easy GUI output folder. +- Forestry preset = `LidarOdometryParams` defaults + the 18 overrides in `lidar_odometry_gui.cpp` "Presets โ†’ 2 โ€ฆ precise forestry". There is no shared preset function; the values are inline GUI code (also copied in the precision_forestry app). + +### Raw Mandeye data (verified on a real file + firmware source) +- LAZ: LAS 1.2 PF1. Real fields: X,Y,Z (sensor frame), intensity (0โ€“160), gps_time, user_data = lidar id. No extra bytes; other standard fields are 0. +- gps_time = Unix epoch seconds (the header wrongly flags adjusted GPS time). +- classification is constant 20 and meaningless: the firmware never copies the Livox tag. A planned firmware fix will put the tag there (bits 0โ€“1 adhesion, 2โ€“3 particles, 4โ€“5 other). +- IMU csv: timestamp (ns, same clock as gps_time), gyroXYZ, accXYZ (g), imuId, timestampUnix (ms, host clock โ€” ignore). 200 Hz. + +### Code (`apps/easy_gui`) +- `easy_gui_core` (static lib, no raylib): `preflight` (file classification mirroring load_data, LAZ/IMU/calibration/disk checks), `staging`, `child_process` (POSIX posix_spawn + process group; Windows CreateProcess + job object, not yet compiled on Windows), `progress_parser` (step 1 log โ†’ phases), `forestry_preset` (writes the step 1 TOML via TomlIO), `export` (M2, see below), `las14_writer` (streaming LAS 1.4 PF6 LAZ writer with a hand-written extra-bytes VLR), `pipeline` (worker thread). Compiles `lidar_odometry_utils.cpp` + `toml_io.cpp` by path with `WITH_GUI=0`; links `core_no_gui` (for `Session`) + `core_math`, not `core`. `core_no_gui` is therefore always built now (core/CMakeLists.txt used to build it only with tests/pybind/CLI tools). +- `easy_gui` (raylib + rlImGui window); `easy_gui --headless ` runs the same pipeline without a window (exit 0/1/2); `easy_gui --export-only ` runs only the export on an existing session (~1.5 min instead of ~22); both take `--check-band ` (test mode, see Export); `--screenshot [recording]` saves the window after ~2 s (raylib saves into the working directory). +- Building `easy_gui` also builds `lidar_odometry_step_1` (`add_dependencies`), which is found next to the executable under its plain or `HDMAPPING_BINARY_PREFIX` name; `HDMAPPING_STEP1_EXECUTABLE` overrides the lookup. +- Output folder: `point_cloud.laz`, `trajectory.csv`, `README.txt` (field/time-base/near-rule description generated with the run's values), `easy_gui.log`, `work/easy_gui_step1.toml`, `work/lio_session/` (kept), `work/staging/` (removed after success unless "keep intermediate files"; all of `work/` is removed after failure or cancel, except that an export failure keeps `work/lio_session/`, since odometry took most of the run). +- Tests: `easy_gui_tests` (doctest). Integration tests against the real step 1 binary run only with `EASY_GUI_TEST_RECORDING=` (e.g. `~/data/mandeye_test/mini4`, symlinks to the first 4 pairs). +- Disk: step 1 needs ~108 B per raw point at peak (cache 7.5 GB + chunk caches ~4 GB + scan_lio for 117 M points). The pre-flight check estimates 120 B per raw point of temporary space + 2.25ร— the raw LAZ size for results (measured: session 0.82ร— + point_cloud.laz 1.24ร— + trajectory.csv 0.03ร— = 2.08ร—), always reports the breakdown, warns below 1.5ร— and refuses below 1ร—. +- Measured on the test recording (117 M raw points, 115 pairs, 10-min walk): step 1 takes ~21 min, peak memory footprint 8.5 GB. Easy GUI (output filter 1.0) vs manual preset 2 (1.5): trajectory **byte-identical** (`session_poses.mrp`, `trajectory_lio_*.csv`), so step 1 is deterministic here and the output filter is confirmed output-only; scan_lio points 89.33 M vs 84.21 M (+5.11 M, +6.1 %). Runs are in `~/data/mandeye_test/runs/{baseline_preset2,easy_full}`. +- Status: M1 done (pre-flight, staging, child-process odometry with progress and cancel). M2 done (export, below). Next: the M3 validation script (`apps/easy_gui/scripts/check_export.py` already covers fields, range, alignment and the step 2 reference; the ramp-time and near-point-density items are still open). + +### Export (M2, `src/export.cpp`) +- Loads the session with `Session` in `load_cache_mode` (poses from `session_poses.mrp` + trajectories, no points: exactly what step 2 uses), then streams one `scan_lio_N.laz` at a time and then one raw LAZ at a time (near points). The near/output-filter thresholds come from the session's own `HDMapping_params_*.toml` (what step 1 ran with). Peak memory: **102 MB** on the full recording; ~1.5 min. +- Odometry points replicate step 2's `save_all_to_las`: `m_pose ยท (p โˆ’ offset) + offset`, gps_time 0 skipped, scale 0.0001 โ†’ XYZ integer-identical to `multi_view_tls_registration_step_2 --export` (89,326,756 points, max |dXYZ| 0, intensity and point_source_id identical). Trajectory CSV matches step 2's `trajectories.csv` (0.05 mm, 5e-10 quaternion). Step 2's own export writes gps_time ร—1e18 (the session's ns ร—1e9 again). +- `range_m` of odometry points: `|calibโปยน ยท csv_pose[node]โปยน ยท p_local|`, node by step 1's rule on the ramp time; verified against the raw range on coincident pairs (p99.9 0.07 mm). 148 odometry points (0.0002 %) have range 70โ€“99.7 m: steep returns (elevation up to 54ยฐ), since step 1's 70 m limit is horizontal. +- Full recording: 89.33 M odometry + 4.04 M near points; 17.1 M of 115.9 M raw points are no-return; point_cloud.laz 907 MB, trajectory.csv 20 MB (114,940 poses). +- Alignment (test mode `--check-band 1.2`, raw points 1.0โ€“1.2 m flagged `near_added = 2`): NN distance to step 1's points median 0.10 mm; with a step 1 counterpart (95.6 %) p90 6.4 mm, p99 12.9 mm; 71 % land on the identical pose. Rebuilding step 1's ramp from the raw files for scans 40โ€“49 reproduces all 7,595,561 step 1 points (โ‰ค 0.17 mm), and shows that the whole difference is step 1's ramp time choosing another node (|ramp โˆ’ raw| p50 1.2 ms, p99 11.6 ms, max 17 ms โ†’ displacement p90 9 mm, max 56 mm). The other 4.4 % are in step 1's inter-chunk gaps (no counterpart). +- `place_by_raw_time` export option (Settings/ExportRequest, `--place-by-raw-time`, GUI advanced checkbox; default **off**): skips scan_lio and places every raw point inside step 1's distance limits from the raw files with the last node before its real gps_time (within 10 ms), writing raw gps_time; near rule unchanged. Full recording: 93,359,291 points with near_added = 0 vs 89,326,756 (+4,026,808 from step 1's inter-chunk gaps, ~5.7 k first-points-of-chunk). Same raw point, default vs raw time (correspondence via rebuilt ramp, both placements verified against the files โ‰ค 0.17 mm): same pose for 79.3 %; median 0 in every bin; p95 12 / 20 / 34 / 69 mm for 1โ€“3 / 3โ€“5 / 5โ€“10 / 10โ€“20 m (moved points only: median 8 / 13 / 23 / 47 mm, p95 19 / 32 / 57 / 118 mm), i.e. a neighbouring node's rotation (~9 mrad). Which is better is not yet measured (the trajectory was optimised with ramp-time points). +- CloudCompare (2.x CLI) loads `range_m` (f32) and `near_added` (u8) as named scalar fields; laspy reads them as float32/uint8 (laszip_add_attribute would set the scale bit and make them float64, hence the hand-written VLR). + +### Test data +`~/data/mandeye_test/continousScanning_0012`: a local copy of `/Volumes/mandeye/continousScanning_0012` without `CAMERA_0`. 116 `lidar*.laz`, 115 `imu*.csv`/`*.sn`/`status*.json`, single lidar, no calibration file. **Never write to `/Volumes/mandeye`.** + +### `load_data` file handling +Sorts all files, then pairs `lidar*.laz` with `imu*.csv` **by position** and loads `min(#laz, #csv)` pairs; a 4-digit index mismatch only prints a warning, and surplus trailing files are skipped. `status*`/`cam*` JSON is ignored; any other `.json` or a `.mjc` is the calibration file; only the first `.sn` is used. Writes `cache/` and `preview/` into the folder holding the input files. + +### Validation items (dev script) +- Easy GUI export vs `multi_view_tls_registration_step_2 --export`: point count, XYZ, intensity, gps_time, trajectory. +- Ramp-time error: for each odometry chunk, rebuild step 1's ramp from the raw points and compare it with their real gps_time. Report it twice: as step 1 does it (no-return (0,0,0) points removed by the load filter *before* the ramp) and with those points kept, to quantify how much the removal contributes. +- Near points: counts added, range histogram, sensor-frame density below 1.5 m. + +### What step 1 does to points (verified in code, `compute_step_1` / `save_result`) +- **Timestamps are not preserved.** In each 20-pose (โ‰ˆ0.1 s) worker chunk, every point's timestamp is overwritten with a uniform ramp `t0 + kยท(t_last โˆ’ t0)/N` (`lidar_odometry.cpp`, "correct points timestamps"). Raw gps_time cannot be joined to output points, with or without a tolerance. +- Output gps_time in `scan_lio_N.laz` and timestamps in `trajectory_lio_N.csv` are seconds ร— 1e9 (ns stored as a double, โ‰ˆ256 ns resolution). +- `scan_lio_N.laz` carries only XYZ (frame of the scan's first trajectory node), intensity and gps_time. Lidar id is dropped in `points_to_vector`. +- Voxel decimation (`params.decimation`) affects only the points used for optimization; `scan_lio_N.laz` is written from the undecimated originals. Points are still dropped by: the load filter (gps_time == 0; horizontal radius outside [filter_threshold_xy_inner, _outer]); points before the first `threshold_initial_points`; the gap `[t_last, t_next_first)` between consecutive worker chunks (โ‰ˆ1 of 20 IMU intervals, ~5 %); the first point of each chunk; chunks with โ‰ค 1000 points; the trailing poses that don't fill a whole chunk; points with sensor-frame norm โ‰ค `threshould_output_filter`; and the last concatenated group if it has โ‰ค `min_counter_concatenated_trajectory_nodes` nodes. +- Distance filters: `filter_threshold_xy_inner/_outer` act at load (`load_point_cloud`), so they remove points from odometry input *and* output; they measure the **horizontal** radius โˆš(xยฒ+yยฒ) in the raw sensor frame. `threshould_output_filter` is **output-only** (`points_to_vector` in `save_result`, `fill_session_from_worker_data`; the optimizers never read it) and measures the **3D** norm in the sensor frame. Consequence: an output filter below `filter_threshold_xy_inner` has no effect, because every surviving point already has 3D norm โ‰ฅ its horizontal radius > inner. +- Per-point range is recoverable without the raw files: `index_pose` = last trajectory node with t_node < t_point (step 1's own rule, applied to the ramp time), then `p_sensor = csv_pose[index_pose]โปยน ยท p_laz` (both are relative to the scan's first node). +- `save_index_pose` is not in TomlIO's attribute list, so it cannot be enabled through a parameter file. + +### Headless step 1 +`lidar_odometry_step_1 ` runs odometry without a window and calls `save_results` + `remove_cache`. Quirks: the exit code is 0 after a caught exception, so check for `session.mjs` instead; `load_data` waits on stdin before `std::exit` when the calibration doesn't match (give the child a closed stdin); `bad_alloc` opens a native message box; progress appears only as log lines (`running iterations: i/n` during setup, `finished optimizing โ€ฆ - i / N โ€ฆ` per odometry chunk). \ No newline at end of file diff --git a/apps/easy_gui/src/easy_gui_main.cpp b/apps/easy_gui/src/easy_gui_main.cpp index 20c7be84..d3f949d2 100644 --- a/apps/easy_gui/src/easy_gui_main.cpp +++ b/apps/easy_gui/src/easy_gui_main.cpp @@ -181,6 +181,13 @@ namespace "Points closer to the scanner than this are left out of the point cloud.\n" "0.4 m (default) removes the operator's body and the handle.\n" "Lower values keep more of the close surroundings but may include the operator."); + ImGui::Checkbox("Place all points by their measurement time (experimental)", &app.settings.place_by_raw_time); + ImGui::SameLine(); + ImGui::TextDisabled("(?)"); + if (ImGui::IsItemHovered()) + ImGui::SetTooltip( + "Off (default): the point cloud uses lidar odometry's own points.\n" + "On: every point is taken from the raw recording and placed at its exact measurement time."); ImGui::Checkbox("Keep intermediate files (for troubleshooting)", &app.settings.keep_work_files); ImGui::Unindent(); } @@ -381,7 +388,7 @@ namespace { //! `easy_gui --headless `: the same pipeline without a window, for scripted runs //! (the dev validation script) and remote machines. Exit code 0 = success, 1 = failed / refused, 2 = usage. - int runHeadless(const fs::path& recording, const fs::path& output_folder, double check_band_outer_m) + int runHeadless(const fs::path& recording, const fs::path& output_folder, const Settings& settings, double check_band_outer_m) { // Unbuffered, so progress shows up as it happens even when redirected to a file or pipe (MSVC treats _IOLBF as // full buffering); the output is a line every few seconds. @@ -399,7 +406,7 @@ namespace Pipeline pipeline; std::string error; - if (!pipeline.start(PipelineRequest{ recording, output_folder, findStep1Executable(), Settings{}, check_band_outer_m }, error)) + if (!pipeline.start(PipelineRequest{ recording, output_folder, findStep1Executable(), settings, check_band_outer_m }, error)) { std::printf("[X] %s\n", error.c_str()); return 1; @@ -436,7 +443,12 @@ namespace //! Dev aid: `easy_gui --export-only [--check-band ]` runs only the //! export, on an existing step 1 session of that recording (step 1 takes ~20 min on a 10-min walk; the export //! about a minute). The output folder may exist; the three result files in it are overwritten. - int runExportOnly(const fs::path& recording, const fs::path& session_file, const fs::path& output_folder, double check_band_outer_m) + int runExportOnly( + const fs::path& recording, + const fs::path& session_file, + const fs::path& output_folder, + const Settings& settings, + double check_band_outer_m) { std::setvbuf(stdout, nullptr, _IONBF, 0); const PreflightResult check = runPreflight(recording, output_folder); @@ -450,8 +462,9 @@ namespace request.session_file = session_file; request.recording = check.files; request.lidar_id = std::max(0, check.lidar_id); - request.near_min_range_m = Settings{}.near_min_range_m; + request.near_min_range_m = settings.near_min_range_m; request.check_band_outer_m = check_band_outer_m; + request.place_by_raw_time = settings.place_by_raw_time; request.laz_file = out.point_cloud; request.trajectory_file = out.trajectory; request.readme_file = out.readme; @@ -478,14 +491,14 @@ namespace return 1; } std::printf( - "[OK] %llu odometry + %llu near + %llu check-band points; raw read %llu, no-return %llu, near outside trajectory %llu; " + "[OK] %llu odometry + %llu near + %llu check-band points; raw read %llu, no-return %llu, outside trajectory %llu; " "range %.3f - %.3f m; %zu poses; LAZ %llu bytes, CSV %llu bytes\n", static_cast(stats.odometry_points), static_cast(stats.near_points), static_cast(stats.check_band_points), static_cast(stats.raw_points_read), static_cast(stats.raw_no_return), - static_cast(stats.raw_near_outside_trajectory), + static_cast(stats.raw_outside_trajectory), stats.range_min_m, stats.range_max_m, stats.trajectory_nodes, @@ -506,6 +519,16 @@ namespace } return 0.0; } + + //! Removes `flag` from the arguments; returns whether it was there. + bool takeFlag(std::vector& args, const std::string& flag) + { + const auto it = std::find(args.begin(), args.end(), flag); + if (it == args.end()) + return false; + args.erase(it); + return true; + } } // namespace int main(int argc, char* argv[]) @@ -514,15 +537,18 @@ int main(int argc, char* argv[]) if (!args.empty() && (args[0] == "--headless" || args[0] == "--export-only")) { const double check_band = takeCheckBand(args); + Settings settings; + settings.place_by_raw_time = takeFlag(args, "--place-by-raw-time"); if (args[0] == "--headless" && args.size() == 3) - return runHeadless(args[1], args[2], check_band); + return runHeadless(args[1], args[2], settings, check_band); if (args[0] == "--export-only" && args.size() == 4) - return runExportOnly(args[1], args[2], args[3], check_band); + return runExportOnly(args[1], args[2], args[3], settings, check_band); std::printf( - "usage: %s --headless [--check-band ]\n" - " %s --export-only [--check-band ]\n" + "usage: %s --headless [--check-band ] [--place-by-raw-time]\n" + " %s --export-only [--check-band ] [--place-by-raw-time]\n" "--check-band (testing): also export the raw points between the 1.0 m load filter and , flagged\n" - " near_added = 2, to check that near points line up with the odometry points.\n", + " near_added = 2, to check that near points line up with the odometry points.\n" + "--place-by-raw-time: place all points from the raw files at their real time (export option, default off).\n", argv[0], argv[0]); return 2; diff --git a/apps/easy_gui/src/export.cpp b/apps/easy_gui/src/export.cpp index 760f9bf5..a21bb581 100644 --- a/apps/easy_gui/src/export.cpp +++ b/apps/easy_gui/src/export.cpp @@ -224,13 +224,16 @@ namespace easy_gui << "All times are Unix epoch seconds (UTC, seconds since 1970-01-01) as counted by the scanner's lidar\n" << "clock. They are NOT GPS week time and NOT adjusted standard GPS time, even though the LAS field is called\n" << "gps_time (the LAS header's global encoding bit 0 is 0 accordingly).\n" - << "Points from lidar odometry (near_added = 0) carry lidar odometry's time, not the raw measurement time:\n" - << "within each 0.1 s piece of the recording, odometry spreads its points' times evenly between the first and\n" - << "last pose of that piece. That is close to the measurement time (typically 1-4 ms off, at most about\n" - << "20 ms), but it cannot be used to match points to the raw files. Odometry also places each point with the\n" - << "pose at that time, so the same offset shows up as small position differences between odometry points\n" - << "and near points: usually a few millimetres, rarely more than 2 cm. Near points (near_added = 1) keep\n" - << "their raw measurement time.\n\n" + << (request.place_by_raw_time + ? "Every point carries its raw measurement time (\"place by raw time\" export option): all points were\n" + "taken from the raw recording and placed with the trajectory pose at that time.\n\n" + : "Points from lidar odometry (near_added = 0) carry lidar odometry's time, not the raw measurement time:\n" + "within each 0.1 s piece of the recording, odometry spreads its points' times evenly between the first and\n" + "last pose of that piece. That is close to the measurement time (typically 1-4 ms off, at most about\n" + "20 ms), but it cannot be used to match points to the raw files. Odometry also places each point with the\n" + "pose at that time, so the same offset shows up as small position differences between odometry points\n" + "and near points: usually a few millimetres, rarely more than 2 cm. Near points (near_added = 1) keep\n" + "their raw measurement time.\n\n") << "Point cloud fields (" << laz << ")\n" << std::string(21 + laz.size(), '-') << "\n" << " X, Y, Z position [m], 0.1 mm resolution\n" @@ -242,7 +245,9 @@ namespace easy_gui << " return_number, number_of_returns always 1; classification always 0 (not classified)\n" << " range_m extra bytes, float32: 3D distance from the lidar at the moment of measurement [m];\n" << " can exceed " << outer << " for steep points, as that limit is horizontal\n" - << " near_added extra bytes, uint8: 0 = point from lidar odometry, 1 = near point added by Easy GUI\n" + << (request.place_by_raw_time + ? " near_added extra bytes, uint8: 0 = point inside lidar odometry's distance limits, 1 = near point\n" + : " near_added extra bytes, uint8: 0 = point from lidar odometry, 1 = near point added by Easy GUI\n") << (stats.check_band_points ? " 2 = TEST MODE alignment-check point (duplicate of an odometry point)\n" : "") << "\n" @@ -259,11 +264,17 @@ namespace easy_gui << " distance at most " << output_filter << ", and horizontal distance below " << outer << ", and\n" << " - it is a real return: points at exactly (0, 0, 0) or with time 0 are \"no return\" records and are\n" << " never exported.\n" - << "Each near point is placed with the trajectory pose lidar odometry uses for points measured at the same\n" - << "time (the last pose before the point), so it lines up with the odometry points. Points measured where\n" + << (request.place_by_raw_time + ? "With the \"place by raw time\" option every point, not only the near points, is taken from the raw\n" + "recording and placed with the last trajectory pose before its measurement time, including the\n" + "points lidar odometry skips between its 0.1 s pieces. " + : "Each near point is placed with the trajectory pose lidar odometry uses for points measured at the same\n" + "time (the last pose before the point), so it lines up with the odometry points. ") + << "Points measured where\n" << "the trajectory has no pose within 10 ms (start and end of the recording) are left out.\n" - << "Near points: " << formatCount(stats.near_points) << ", odometry points: " << formatCount(stats.odometry_points) - << ".\n\n" + << "Near points: " << formatCount(stats.near_points) + << (request.place_by_raw_time ? ", points inside lidar odometry's limits: " : ", odometry points: ") + << formatCount(stats.odometry_points) << ".\n\n" << "Trajectory fields (" << csv << ", comma-separated, one header row)\n" << std::string(53 + csv.size(), '-') << "\n" << " timestamp_unix_s time of the pose [Unix epoch seconds], same clock as the point cloud's gps_time\n" @@ -398,7 +409,8 @@ namespace easy_gui return false; } const uint8_t user_data = static_cast(request.lidar_id); - const double work_units = static_cast(clouds.size() + request.recording.pairs); + const size_t scans_to_copy = request.place_by_raw_time ? 0 : clouds.size(); + const double work_units = static_cast(scans_to_copy + request.recording.pairs); auto writePoint = [&](const Eigen::Vector3d& position, double time_s, uint16_t intensity, size_t scan, double range, NearFlag flag) { const ExtraBytes extra{ static_cast(range), flag }; @@ -414,8 +426,8 @@ namespace easy_gui return writer.write(p, error); }; - // --- 1. step 1's points, one scan at a time - for (size_t i = 0; i < clouds.size(); i++) + // --- 1. step 1's points, one scan at a time (not with place_by_raw_time: pass 2 places them from the raw files) + for (size_t i = 0; i < scans_to_copy; i++) { if (cancelled()) return false; @@ -465,7 +477,7 @@ namespace easy_gui } } - // --- 2. near points from the raw recording, one file at a time + // --- 2. near points (with place_by_raw_time: all points) from the raw recording, one file at a time const double inner = params->filter_threshold_xy_inner; const double outer = params->filter_threshold_xy_outer; const double output_filter = params->threshould_output_filter; @@ -474,8 +486,9 @@ namespace easy_gui if (cancelled()) return false; progress( - static_cast(clouds.size() + f) / work_units, - "Adding the near points (file " + std::to_string(f + 1) + " of " + std::to_string(request.recording.pairs) + ")"); + static_cast(scans_to_copy + f) / work_units, + std::string(request.place_by_raw_time ? "Writing the point cloud" : "Adding the near points") + " (file " + + std::to_string(f + 1) + " of " + std::to_string(request.recording.pairs) + ")"); const fs::path& raw_file = request.recording.laz_files[f]; LazReader reader; @@ -501,17 +514,23 @@ namespace easy_gui const Eigen::Vector3d lidar = reader.position(); const double range_xy = std::hypot(lidar.x(), lidar.y()); const double range = lidar.norm(); - if (range < request.near_min_range_m || range_xy >= outer) - continue; const Eigen::Vector3d sensor = calibration * lidar; - // Exactly the complement of step 1's distance filters: load_point_cloud keeps - // inner < range_xy < outer, points_to_vector keeps |sensor| > threshould_output_filter. + // Step 1's distance filters: load_point_cloud keeps inner < range_xy < outer, points_to_vector keeps + // |sensor| > threshould_output_filter. Near points are exactly the complement, at close range. + const bool step1_keeps = range_xy > inner && range_xy < outer && sensor.norm() > output_filter; NearFlag flag; - if (range_xy <= inner || sensor.norm() <= output_filter) + if (step1_keeps) + { + if (request.place_by_raw_time) + flag = kFromOdometry; + else if (range_xy < request.check_band_outer_m) + flag = kCheckBand; + else + continue; // exported from scan_lio_N.laz in pass 1 + } + else if (range >= request.near_min_range_m && range_xy < outer) flag = kNearAdded; - else if (range_xy < request.check_band_outer_m) - flag = kCheckBand; else continue; @@ -519,7 +538,7 @@ namespace easy_gui const auto it = std::lower_bound(node_times.begin(), node_times.end(), t); if (it == node_times.begin() || t - *(it - 1) > kNodeTolerance_s) { - stats.raw_near_outside_trajectory++; + stats.raw_outside_trajectory++; continue; } const Node& node = nodes[static_cast(it - node_times.begin()) - 1]; diff --git a/apps/easy_gui/src/export.h b/apps/easy_gui/src/export.h index b507cdec..564d8c3e 100644 --- a/apps/easy_gui/src/export.h +++ b/apps/easy_gui/src/export.h @@ -28,6 +28,11 @@ namespace easy_gui //! Test mode: > 0 also adds the raw points with filter_threshold_xy_inner < horizontal range < this, which step 1 //! exported too, flagged kCheckBand, so their placement can be compared with step 1's own points. double check_band_outer_m = 0.0; + //! Place every point from the raw files with the trajectory pose at its real gps_time, instead of taking step + //! 1's points (scan_lio_N.laz, placed at step 1's ramp time). Same distance filters and near-point rule; raw + //! gps_time is written; includes the raw points step 1 drops between its 0.1 s pieces. near_added = 0 then + //! means "inside step 1's distance limits". check_band_outer_m is ignored (those points are all included). + bool place_by_raw_time = false; std::filesystem::path laz_file; std::filesystem::path trajectory_file; std::filesystem::path readme_file; @@ -42,7 +47,7 @@ namespace easy_gui uint64_t check_band_points = 0; // near_added = 2 uint64_t raw_points_read = 0; uint64_t raw_no_return = 0; // (0,0,0) or gps_time 0: never exported - uint64_t raw_near_outside_trajectory = 0; // near candidates with no trajectory node within 10 ms + uint64_t raw_outside_trajectory = 0; // exported candidates with no trajectory node within 10 ms double range_min_m = 0.0; double range_max_m = 0.0; //! range_m histogram, 0.1 m bins from 0 m; the last bin collects everything beyond. diff --git a/apps/easy_gui/src/pipeline.cpp b/apps/easy_gui/src/pipeline.cpp index ada37e7c..618785dc 100644 --- a/apps/easy_gui/src/pipeline.cpp +++ b/apps/easy_gui/src/pipeline.cpp @@ -216,7 +216,7 @@ namespace easy_gui { std::ostringstream s; s << "[easy_gui] settings: near_min_range_m=" << request_.settings.near_min_range_m - << " keep_work_files=" << request_.settings.keep_work_files; + << " place_by_raw_time=" << request_.settings.place_by_raw_time << " keep_work_files=" << request_.settings.keep_work_files; log(s.str()); } for (const auto& f : check.findings) @@ -329,6 +329,7 @@ namespace easy_gui request.lidar_id = std::max(0, check.lidar_id); request.near_min_range_m = request_.settings.near_min_range_m; request.check_band_outer_m = request_.check_band_outer_m; + request.place_by_raw_time = request_.settings.place_by_raw_time; request.laz_file = out.point_cloud; request.trajectory_file = out.trajectory; request.readme_file = out.readme; @@ -376,7 +377,7 @@ namespace easy_gui if (stats.check_band_points) s << " + " << stats.check_band_points << " check-band points"; s << " (" << stats.raw_no_return << " of " << stats.raw_points_read << " raw points were no-return, " - << stats.raw_near_outside_trajectory << " near candidates outside the trajectory), range " << stats.range_min_m << " - " + << stats.raw_outside_trajectory << " candidates outside the trajectory), range " << stats.range_min_m << " - " << stats.range_max_m << " m, " << stats.trajectory_nodes << " trajectory poses; point cloud " << stats.laz_bytes << " bytes, trajectory " << stats.trajectory_bytes << " bytes"; log(s.str()); diff --git a/apps/easy_gui/src/settings.h b/apps/easy_gui/src/settings.h index be9a293d..01a64c6d 100644 --- a/apps/easy_gui/src/settings.h +++ b/apps/easy_gui/src/settings.h @@ -13,6 +13,10 @@ namespace easy_gui static constexpr double kNearMinRangeLowest = 0.1; static constexpr double kNearMinRangeHighest = 0.95; // must stay below the 1.0 m load filter + //! Export option (default off while it is being evaluated): place all points from the raw files at their real + //! time instead of using lidar odometry's points (ExportRequest::place_by_raw_time). + bool place_by_raw_time = false; + //! Keep the step 1 staging/preview folders after a successful run (for debugging). bool keep_work_files = false; diff --git a/apps/easy_gui/tests/test_export.cpp b/apps/easy_gui/tests/test_export.cpp index 11ef8418..f227de44 100644 --- a/apps/easy_gui/tests/test_export.cpp +++ b/apps/easy_gui/tests/test_export.cpp @@ -368,7 +368,7 @@ TEST_CASE("exportSession: step 1 points as step 2 places them, near points with CHECK(stats.check_band_points == 0); CHECK(stats.raw_points_read == file0.size() + file1.size()); CHECK(stats.raw_no_return == 2); - CHECK(stats.raw_near_outside_trajectory == 2); + CHECK(stats.raw_outside_trajectory == 2); CHECK(stats.filter_threshold_xy_inner == 1.0); CHECK(stats.threshould_output_filter == 1.0); // Easy GUI's output setting, read back from the session @@ -457,6 +457,35 @@ TEST_CASE("exportSession: step 1 points as step 2 places them, near points with CHECK(c.range_m == doctest::Approx(band.norm())); } + SUBCASE("place_by_raw_time takes every point from the raw files, at its raw time") + { + request.place_by_raw_time = true; + request.check_band_outer_m = 1.2; // ignored in this mode + REQUIRE(exportSession( + request, + cancel, + [](double, const std::string&) + { + }, + stats, + error)); + CHECK(stats.odometry_points == 2); // the 3 m point and the 1.1 m point, not scan_lio's points + CHECK(stats.near_points == 2); + CHECK(stats.check_band_points == 0); + CHECK(stats.raw_outside_trajectory == 2); + const ReadFile rr = readLaz(request.laz_file); + REQUIRE(rr.points.size() == 4); // raw file order: near_a, 3 m, band, near_steep + CHECK(rr.points[0].near_added == kNearAdded); + CHECK(rr.points[1].near_added == kFromOdometry); + CHECK(near(rr.points[1].xyz, S::pose(5) * Eigen::Vector3d(3.0, 0.0, 0.0))); + CHECK(rr.points[1].gps_time == t5); + CHECK(rr.points[1].range_m == doctest::Approx(3.0)); + CHECK(rr.points[2].near_added == kFromOdometry); + CHECK(near(rr.points[2].xyz, S::pose(5) * band)); + CHECK(rr.points[3].near_added == kNearAdded); + CHECK(rr.points[3].point_source_id == 1); + } + SUBCASE("cancel stops the export") { cancel = true; From 935bf03f6fdeea81693a9b177aab9c27c343b2a2 Mon Sep 17 00:00:00 2001 From: Martin Mokros Date: Sat, 17 Oct 2026 21:38:10 +0100 Subject: [PATCH 04/23] Easy GUI export: add step 1's skipped gap points; point_origin replaces near_added The default export is now a hybrid: step 1's own points as step 1 placed them, plus two kinds of raw points step 1 left out, both placed with the pose at their real gps_time: - near points (unchanged rule), and - gap points: raw points inside step 1's distance limits that step 1 skips between its 0.1 s pieces (from a piece's last pose to the next piece's first). On the 117 M-point test walk that is 4,026,788 points (4.1 %). The extra-bytes flag near_added becomes point_origin (uint8): 0 = step 1, 1 = near point, 2 = gap point, 3 = check-band test point. The README now explains where each kind of point comes from. Validation (check_export.py, full recording): gap points sit as close to step 1's points as a missing 5 ms slice of step 1's own points does (NN median 19.4 vs 19.8 mm, p99 187 vs 192 mm, the same in every range bin); step 1's points are still integer-identical to step 2's export. Peak memory 106 MB; point_cloud.laz 948 MB (1.29x the raw LAZ, within the 2.25x pre-flight estimate). place_by_raw_time stays an experimental option (default off): on the same 89.3 M matched points it measured slightly less sharp than step 1's own placement (ground +0.1 mm median, thinner in only 37 % of cells; stem RMSE +0.09 mm, sharper for 3 of 17 trunks; no diameter bias). The sharpness study is recorded in CLAUDE.md. Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 17 +- apps/easy_gui/scripts/check_export.py | 115 ++++++++++---- apps/easy_gui/src/easy_gui_main.cpp | 11 +- apps/easy_gui/src/export.cpp | 146 +++++++++++------- apps/easy_gui/src/export.h | 30 ++-- apps/easy_gui/src/las14_writer.cpp | 2 +- apps/easy_gui/src/pipeline.cpp | 3 +- apps/easy_gui/src/preflight.cpp | 4 +- apps/easy_gui/src/preflight.h | 2 +- apps/easy_gui/tests/test_export.cpp | 72 ++++++--- .../tests/test_pipeline_integration.cpp | 1 + 11 files changed, 261 insertions(+), 142 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 7340d734..336fffe2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -89,8 +89,9 @@ Constraints: raylib stack; reuse existing code, don't duplicate algorithms; don' Decisions: - v1 = odometry โ†’ export, no refinement. Validate it against a manual step 1 โ†’ step 2 run first. v2 adds gated pose-graph loop closure (`PoseGraphLoopClosure`, auto-detected candidates, edges rejected if ICP fails) behind a toggle. Do not use `PoseGraphSLAM::optimize`. - Output LAZ: LAS 1.4, point format 6, extra-bytes attributes. `gps_time` = Unix epoch seconds; global encoding bit 0 (GPS time type) = 0, since it is not adjusted GPS time. Document the time base. +- Default export = hybrid: step 1's own points as step 1 placed them (`point_origin = 0`), plus near points (1) and gap points (2), both from the raw files and placed by real gps_time. Gap points = raw points inside step 1's distance limits that step 1 skipped between its 0.1 s pieces: the last trajectory node at or before t is a piece's last node (`j % threshold_nr_poses == threshold_nr_poses โˆ’ 1` within the scan; step 1 takes [first, last node) of each piece). `point_origin = 3` = check-band test points. Placing step 1's own points by raw time is only an experimental option (`place_by_raw_time`), because it measured slightly less sharp (see Export). - Multi-lidar rigs are out of scope for v1; the pre-flight check rejects recordings with more than one lidar id. -- Near points (added by Easy GUI at export, never seen by odometry): keep raw points with `near_min_range` โ‰ค 3D range **and** that step 1's distance filters removed. As implemented, that is the exact complement: horizontal range โ‰ค `filter_threshold_xy_inner` (1.0 m) or calibrated 3D norm โ‰ค `threshould_output_filter`, and horizontal range < `filter_threshold_xy_outer`. Always drop exact (0,0,0) and gps_time 0 no-return points (~15 % of raw points). Transform each one with step 1's rule at its *raw* gps_time: the last trajectory node with t_node < t, if within 10 ms (step 1's own tolerance), so `global = mrp[scan] ยท csv_pose[node] ยท calib ยท p_raw`. Mark it with an extra-bytes `near_added` flag (uint8; `user_data` keeps the lidar id). `near_min_range` is an advanced setting, default 0.4 m: measured raw data shows the operator's body/handle only as small blobs inside 0.4 m, and near points beyond that are environment in all directions. +- Near points (added by Easy GUI at export, never seen by odometry): keep raw points with `near_min_range` โ‰ค 3D range **and** that step 1's distance filters removed. As implemented, that is the exact complement: horizontal range โ‰ค `filter_threshold_xy_inner` (1.0 m) or calibrated 3D norm โ‰ค `threshould_output_filter`, and horizontal range < `filter_threshold_xy_outer`. Always drop exact (0,0,0) and gps_time 0 no-return points (~15 % of raw points). Transform each one with step 1's rule at its *raw* gps_time: the last trajectory node with t_node < t, if within 10 ms (step 1's own tolerance), so `global = mrp[scan] ยท csv_pose[node] ยท calib ยท p_raw`. Mark it `point_origin = 1` (extra bytes, uint8; `user_data` keeps the lidar id). `near_min_range` is an advanced setting, default 0.4 m: measured raw data shows the operator's body/handle only as small blobs inside 0.4 m, and near points beyond that are environment in all directions. - For the near rule to meet step 1's output without a gap, Easy GUI writes `threshould_output_filter = filter_threshold_xy_inner` (1.0 instead of the preset's 1.5). It is output-only, so the trajectory is unchanged; points with horizontal range โ‰ฅ 1.0 m then all come from step 1, and points below that come from the near rule. - Easy GUI never writes into the recording folder (it may be on the Mandeye USB stick). `load_data` puts `cache/` and `preview/` next to the input files, so step 1 runs on a staging folder of symlinks inside the Easy GUI output folder. - Forestry preset = `LidarOdometryParams` defaults + the 18 overrides in `lidar_odometry_gui.cpp` "Presets โ†’ 2 โ€ฆ precise forestry". There is no shared preset function; the values are inline GUI code (also copied in the precision_forestry app). @@ -107,7 +108,7 @@ Decisions: - Building `easy_gui` also builds `lidar_odometry_step_1` (`add_dependencies`), which is found next to the executable under its plain or `HDMAPPING_BINARY_PREFIX` name; `HDMAPPING_STEP1_EXECUTABLE` overrides the lookup. - Output folder: `point_cloud.laz`, `trajectory.csv`, `README.txt` (field/time-base/near-rule description generated with the run's values), `easy_gui.log`, `work/easy_gui_step1.toml`, `work/lio_session/` (kept), `work/staging/` (removed after success unless "keep intermediate files"; all of `work/` is removed after failure or cancel, except that an export failure keeps `work/lio_session/`, since odometry took most of the run). - Tests: `easy_gui_tests` (doctest). Integration tests against the real step 1 binary run only with `EASY_GUI_TEST_RECORDING=` (e.g. `~/data/mandeye_test/mini4`, symlinks to the first 4 pairs). -- Disk: step 1 needs ~108 B per raw point at peak (cache 7.5 GB + chunk caches ~4 GB + scan_lio for 117 M points). The pre-flight check estimates 120 B per raw point of temporary space + 2.25ร— the raw LAZ size for results (measured: session 0.82ร— + point_cloud.laz 1.24ร— + trajectory.csv 0.03ร— = 2.08ร—), always reports the breakdown, warns below 1.5ร— and refuses below 1ร—. +- Disk: step 1 needs ~108 B per raw point at peak (cache 7.5 GB + chunk caches ~4 GB + scan_lio for 117 M points). The pre-flight check estimates 120 B per raw point of temporary space + 2.25ร— the raw LAZ size for results (measured: session 0.82ร— + point_cloud.laz 1.29ร— with near and gap points + trajectory.csv 0.03ร— = 2.14ร—), always reports the breakdown, warns below 1.5ร— and refuses below 1ร—. - Measured on the test recording (117 M raw points, 115 pairs, 10-min walk): step 1 takes ~21 min, peak memory footprint 8.5 GB. Easy GUI (output filter 1.0) vs manual preset 2 (1.5): trajectory **byte-identical** (`session_poses.mrp`, `trajectory_lio_*.csv`), so step 1 is deterministic here and the output filter is confirmed output-only; scan_lio points 89.33 M vs 84.21 M (+5.11 M, +6.1 %). Runs are in `~/data/mandeye_test/runs/{baseline_preset2,easy_full}`. - Status: M1 done (pre-flight, staging, child-process odometry with progress and cancel). M2 done (export, below). Next: the M3 validation script (`apps/easy_gui/scripts/check_export.py` already covers fields, range, alignment and the step 2 reference; the ramp-time and near-point-density items are still open). @@ -115,10 +116,14 @@ Decisions: - Loads the session with `Session` in `load_cache_mode` (poses from `session_poses.mrp` + trajectories, no points: exactly what step 2 uses), then streams one `scan_lio_N.laz` at a time and then one raw LAZ at a time (near points). The near/output-filter thresholds come from the session's own `HDMapping_params_*.toml` (what step 1 ran with). Peak memory: **102 MB** on the full recording; ~1.5 min. - Odometry points replicate step 2's `save_all_to_las`: `m_pose ยท (p โˆ’ offset) + offset`, gps_time 0 skipped, scale 0.0001 โ†’ XYZ integer-identical to `multi_view_tls_registration_step_2 --export` (89,326,756 points, max |dXYZ| 0, intensity and point_source_id identical). Trajectory CSV matches step 2's `trajectories.csv` (0.05 mm, 5e-10 quaternion). Step 2's own export writes gps_time ร—1e18 (the session's ns ร—1e9 again). - `range_m` of odometry points: `|calibโปยน ยท csv_pose[node]โปยน ยท p_local|`, node by step 1's rule on the ramp time; verified against the raw range on coincident pairs (p99.9 0.07 mm). 148 odometry points (0.0002 %) have range 70โ€“99.7 m: steep returns (elevation up to 54ยฐ), since step 1's 70 m limit is horizontal. -- Full recording: 89.33 M odometry + 4.04 M near points; 17.1 M of 115.9 M raw points are no-return; point_cloud.laz 907 MB, trajectory.csv 20 MB (114,940 poses). -- Alignment (test mode `--check-band 1.2`, raw points 1.0โ€“1.2 m flagged `near_added = 2`): NN distance to step 1's points median 0.10 mm; with a step 1 counterpart (95.6 %) p90 6.4 mm, p99 12.9 mm; 71 % land on the identical pose. Rebuilding step 1's ramp from the raw files for scans 40โ€“49 reproduces all 7,595,561 step 1 points (โ‰ค 0.17 mm), and shows that the whole difference is step 1's ramp time choosing another node (|ramp โˆ’ raw| p50 1.2 ms, p99 11.6 ms, max 17 ms โ†’ displacement p90 9 mm, max 56 mm). The other 4.4 % are in step 1's inter-chunk gaps (no counterpart). -- `place_by_raw_time` export option (Settings/ExportRequest, `--place-by-raw-time`, GUI advanced checkbox; default **off**): skips scan_lio and places every raw point inside step 1's distance limits from the raw files with the last node before its real gps_time (within 10 ms), writing raw gps_time; near rule unchanged. Full recording: 93,359,291 points with near_added = 0 vs 89,326,756 (+4,026,808 from step 1's inter-chunk gaps, ~5.7 k first-points-of-chunk). Same raw point, default vs raw time (correspondence via rebuilt ramp, both placements verified against the files โ‰ค 0.17 mm): same pose for 79.3 %; median 0 in every bin; p95 12 / 20 / 34 / 69 mm for 1โ€“3 / 3โ€“5 / 5โ€“10 / 10โ€“20 m (moved points only: median 8 / 13 / 23 / 47 mm, p95 19 / 32 / 57 / 118 mm), i.e. a neighbouring node's rotation (~9 mrad). Which is better is not yet measured (the trajectory was optimised with ramp-time points). -- CloudCompare (2.x CLI) loads `range_m` (f32) and `near_added` (u8) as named scalar fields; laspy reads them as float32/uint8 (laszip_add_attribute would set the scale bit and make them float64, hence the hand-written VLR). +- Full recording (hybrid default): 89,326,756 step 1 + 4,038,530 near + 4,026,788 gap points = 97.39 M; 17.1 M of 115.9 M raw points are no-return; point_cloud.laz 948 MB, trajectory.csv 20 MB (114,940 poses); 143 s, peak 106 MB. +- Gap points, validated like the near points (`check_export.py`, per scan against step 1's points of scans sโˆ’1..s+1): NN distance median 19.4 mm, p99 187 mm, vs a control of step 1's own points from the 5 ms before each gap queried without them: median 19.8 mm, p99 192 mm; the same in every range bin (1โ€“3 m 10.6 vs 10.9 mm โ€ฆ 20+ m 68.1 vs 68.2 mm). So the gap points sit where a missing slice of step 1's own points would. 277 gap points are within 0.5 ยตs of a pose time and need a 1 ยตs tolerance when classified from trajectory.csv (1 ยตs resolution). +- Alignment (test mode `--check-band 1.2`, raw points 1.0โ€“1.2 m, now `point_origin = 3`): NN distance to step 1's points median 0.10 mm; with a step 1 counterpart (95.6 %) p90 6.4 mm, p99 12.9 mm; 71 % land on the identical pose. Rebuilding step 1's ramp from the raw files for scans 40โ€“49 reproduces all 7,595,561 step 1 points (โ‰ค 0.17 mm), and shows that the whole difference is step 1's ramp time choosing another node (|ramp โˆ’ raw| p50 1.2 ms, p99 11.6 ms, max 17 ms โ†’ displacement p90 9 mm, max 56 mm). The other 4.4 % are in step 1's inter-chunk gaps (no counterpart). +- `place_by_raw_time` export option (Settings/ExportRequest, `--place-by-raw-time`, GUI advanced checkbox; default **off**): skips scan_lio and places every raw point inside step 1's distance limits from the raw files with the last node before its real gps_time (within 10 ms), writing raw gps_time; near rule unchanged. Full recording: 93,359,291 points inside step 1's limits vs 89,326,756 (+4,026,808 from step 1's inter-chunk gaps, ~5.7 k first-points-of-chunk). Same raw point, default vs raw time (correspondence via rebuilt ramp, both placements verified against the files โ‰ค 0.17 mm): same pose for 79.3 %; median 0 in every bin; p95 12 / 20 / 34 / 69 mm for 1โ€“3 / 3โ€“5 / 5โ€“10 / 10โ€“20 m (moved points only: median 8 / 13 / 23 / 47 mm, p95 19 / 32 / 57 / 118 mm), i.e. a neighbouring node's rotation (~9 mrad). Sharpness (2026-10-17, the same 89,326,756 matched points, selections made on the mean of both positions): + - Ground, 1 ร— 1 m cells within 20 m of the walk, lowest points ยฑ0.15 m band, per-version orthogonal plane fit (5,376 cells): residual std median 65.73 mm default vs 66.14 mm raw time; paired difference (raw โˆ’ default) +0.06 / +0.08 / +0.15 / +0.61 mm at 0โ€“3 / 3โ€“5 / 5โ€“10 / 10โ€“20 m from the path; raw time thinner in only 37 % of cells (Wilcoxon p โ‰ˆ 1e-112). A ยฑ0.05 m band gives the same sign (41 % thinner, +0.05 โ€ฆ +0.24 mm). The ~66 mm is mostly forest-floor roughness, which the paired design cancels. + - Stems, 17 auto-detected and visually checked trunks (1.2โ€“1.4 m above local ground, circle fit to the same points in each version): RMSE median 19.25 vs 19.31 mm, paired +0.05 / +0.43 / +0.03 / +0.17 mm at 1โ€“3 / 3โ€“5 / 5โ€“10 / 10โ€“20 m; raw time sharper for 3 of 17 (p = 0.06). Diameters do not differ systematically: median |ฮ”D| 0.4 mm (< 2 mm within 10 m; โˆ’3 โ€ฆ +7 mm for one-sided arcs at 10โ€“20 m). + - Verdict: default (step 1 placement) slightly sharper, by 0.1โ€“0.6 mm on 20โ€“60 mm thick surfaces; most likely because step 1 optimised the trajectory with its ramp-time points. Scripts: `ground.py`, `stems.py` (not in the repo). +- CloudCompare (2.x CLI) loaded `range_m` (f32) and `near_added` (u8, renamed `point_origin` since) as named scalar fields; laspy reads them as float32/uint8 (laszip_add_attribute would set the scale bit and make them float64, hence the hand-written VLR). ### Test data `~/data/mandeye_test/continousScanning_0012`: a local copy of `/Volumes/mandeye/continousScanning_0012` without `CAMERA_0`. 116 `lidar*.laz`, 115 `imu*.csv`/`*.sn`/`status*.json`, single lidar, no calibration file. **Never write to `/Volumes/mandeye`.** diff --git a/apps/easy_gui/scripts/check_export.py b/apps/easy_gui/scripts/check_export.py index 1bbf7a8c..b78eb59b 100644 --- a/apps/easy_gui/scripts/check_export.py +++ b/apps/easy_gui/scripts/check_export.py @@ -6,12 +6,16 @@ Needs laspy with a LAZ backend, numpy and scipy (pip install "laspy[lazrs]" numpy scipy). Checks: - fields LAS 1.4 / PF6, GPS time type bit, named extra bytes (range_m float32, near_added uint8), + fields LAS 1.4 / PF6, GPS time type bit, named extra bytes (range_m float32, point_origin uint8), point_source_id = scan index, user_data = lidar id, gps_time is Unix seconds. range histogram of range_m; nothing below the near limit, nothing at the sensor origin. alignment only with an export made with --check-band: nearest-neighbour distance from each check-band point - (raw point placed the near-point way) to step 1's own points. A few mm median = same transform chain. - reference with --step2: the near_added = 0 points against step 2's full-resolution export of the same session + (point_origin 3: raw point placed the near-point way) to step 1's own points. A few mm median = same + transform chain. + gap points nearest-neighbour distance from each gap point (point_origin 2) to step 1's points of the same and the + neighbouring scans, next to a control: step 1's own points from the 5 ms before each gap, queried against + step 1's points without them (a correctly placed missing slice). + reference with --step2: the point_origin = 0 points against step 2's full-resolution export of the same session (multi_view_tls_registration_step_2 --export ): count, XYZ, intensity, trajectory. Exit code 1 if a check fails. """ @@ -41,9 +45,9 @@ def check_fields(laz): check(str(h.version) == "1.4" and h.point_format.id == 6, f"LAS {h.version}, point format {h.point_format.id}") check(not h.global_encoding.gps_time_type, "global encoding bit 0 (GPS time type) = 0") extra = {d.name: d for d in h.point_format.extra_dimensions} - check(set(extra) == {"range_m", "near_added"}, f"extra bytes: {sorted(extra)}") + check(set(extra) == {"range_m", "point_origin"}, f"extra bytes: {sorted(extra)}") check(np.dtype(extra["range_m"].dtype) == np.float32, f"range_m is {np.dtype(extra['range_m'].dtype)}") - check(np.dtype(extra["near_added"].dtype) == np.uint8, f"near_added is {np.dtype(extra['near_added'].dtype)}") + check(np.dtype(extra["point_origin"].dtype) == np.uint8, f"point_origin is {np.dtype(extra['point_origin'].dtype)}") print(f" {h.point_count:,} points, bounds {h.mins.round(1)} .. {h.maxs.round(1)}, {h.generating_software}") @@ -51,15 +55,15 @@ def scan(laz): """One streaming pass: per-flag counts, range histogram, sensor-origin and time checks, and the points needed for the alignment check (step 1 points up to 3 m + check-band points), all with bounded memory. 3 m, not the band's 1.2 m: the band limits the horizontal range, steep points reach ~2 m in 3D.""" - stats = {"count": np.zeros(3, np.int64), "hist": np.zeros(1201, np.int64), "range_min": [np.inf] * 3, - "range_max": [-np.inf] * 3, "t_min": np.inf, "t_max": -np.inf, "psid": set(), "user_data": set(), + stats = {"count": np.zeros(4, np.int64), "hist": np.zeros(1201, np.int64), "range_min": [np.inf] * 4, + "range_max": [-np.inf] * 4, "t_min": np.inf, "t_max": -np.inf, "psid": set(), "user_data": set(), "origin": 0, "close_odo": [], "band": [], "big_range": []} with laspy.open(laz) as f: for p in f.chunk_iterator(CHUNK): - flag = np.asarray(p.near_added) + flag = np.asarray(p.point_origin) r = np.asarray(p.range_m) xyz = np.column_stack([p.x, p.y, p.z]) - for k in range(3): + for k in range(4): m = flag == k if m.any(): stats["count"][k] += m.sum() @@ -73,7 +77,7 @@ def scan(laz): stats["origin"] += int((np.abs(xyz).sum(axis=1) == 0).sum()) + int((r == 0).sum()) m = (flag == 0) & (r < 3.0) stats["close_odo"].append((xyz[m], r[m])) - stats["band"].append((xyz[flag == 2], np.asarray(p.gps_time)[flag == 2])) + stats["band"].append(xyz[flag == 3]) far = r > 70 if far.any(): stats["big_range"].append(np.column_stack([r[far], xyz[far, 2], np.asarray(p.point_source_id)[far]])) @@ -82,8 +86,8 @@ def scan(laz): def check_range(stats, near_min): print("range") - names = ["odometry (0)", "near added (1)", "check band (2)"] - for k in range(3): + names = ["step 1 (0)", "near point (1)", "gap point (2)", "check band (3)"] + for k in range(4): if stats["count"][k]: print(f" {names[k]:15s} {stats['count'][k]:>12,} points, range_m {stats['range_min'][k]:.3f} .. {stats['range_max'][k]:.3f} m") check(min(stats["range_min"]) >= near_min - 1e-6, f"no range_m below {near_min} m (min {min(stats['range_min']):.4f})") @@ -103,40 +107,82 @@ def check_range(stats, near_min): f"z {big[:, 1].min():.1f} .. {big[:, 1].max():.1f} m)") -def check_alignment(stats, folder): - band = np.concatenate([b[0] for b in stats["band"]]) if stats["band"] else np.empty((0, 3)) +def check_alignment(stats): + band = np.concatenate(stats["band"]) if stats["band"] else np.empty((0, 3)) if len(band) == 0: return print("alignment (check-band points vs step 1 points)") - band_t = np.concatenate([b[1] for b in stats["band"]]) - band_r_max = stats["range_max"][2] + band_r_max = stats["range_max"][3] odo = np.concatenate([o[0][o[1] <= band_r_max + 0.01] for o in stats["close_odo"]]) - d, _ = cKDTree(odo).query(band, k=1, workers=-1) - d_mm = d * 1000 - # Step 1 drops the raw points between the last pose of each 20-pose worker chunk and the next chunk's first pose - # (~5 % of the time); check-band points from there have no step 1 counterpart, so they are reported separately. + d_mm = cKDTree(odo).query(band, k=1, workers=-1)[0] * 1000 + q = np.percentile(d_mm, [50, 90, 99]) + print(f" {len(band):,} check-band points vs {len(odo):,} step 1 points up to {band_r_max:.2f} m") + print(f" NN distance: median {q[0]:.2f} mm, p90 {q[1]:.2f}, p99 {q[2]:.2f}, max {d_mm.max():.1f} mm") + print(f" exactly coincident (< 0.15 mm, same trajectory pose as step 1): {100 * (d_mm < 0.15).mean():.1f} %") + check(q[0] < 5.0, f"median NN distance {q[0]:.2f} mm < 5 mm") + + +def check_gaps(laz, folder): + """Gap points vs step 1's points, per scan (tree: step 1 points of scans s-1..s+1), next to a control: step 1's + points from the last node interval before each gap ([node 18, node 19) of every 20-node piece), queried against the + same tree without them.""" + print("gap points (vs step 1 points)") + parts = [] + with laspy.open(laz) as f: + for p in f.chunk_iterator(CHUNK): + o = np.asarray(p.point_origin) + m = (o == 0) | (o == 2) + parts.append((np.column_stack([p.x[m], p.y[m], p.z[m]]).astype(np.float32), o[m], np.asarray(p.point_source_id)[m], + np.asarray(p.gps_time)[m], np.asarray(p.range_m)[m])) + xyz, origin, psid, t, r = (np.concatenate([q[i] for q in parts]) for i in range(5)) + del parts + if not (origin == 2).any(): + check(False, "gap points present") + return traj = np.genfromtxt(folder / "trajectory.csv", delimiter=",", names=True, usecols=(0, 11)) node_in_scan = np.zeros(len(traj), np.int64) for s in np.unique(traj["scan_index"]): m = np.where(traj["scan_index"] == s)[0] node_in_scan[m] = np.arange(len(m)) - node = np.searchsorted(traj["timestamp_unix_s"], band_t, side="left") - 1 - in_gap = node_in_scan[node] % 20 == 19 + node = np.searchsorted(traj["timestamp_unix_s"], t, side="left") - 1 + control = (origin == 0) & (node_in_scan[np.maximum(node, 0)] % 20 == 18) + # trajectory.csv has 1 us time resolution, so points within 0.5 us of a pose may land on either side of it + tg = t[origin == 2] + at_or_before = np.searchsorted(traj["timestamp_unix_s"], tg + 1e-6, side="right") - 1 + before = np.searchsorted(traj["timestamp_unix_s"], tg - 1e-6, side="left") - 1 + piece_end = (node_in_scan[at_or_before] % 20 == 19) | (node_in_scan[np.maximum(before, 0)] % 20 == 19) + exact = node_in_scan[np.maximum(node[origin == 2], 0)] % 20 == 19 + check(np.all(piece_end), f"every gap point lies after the last pose of a step 1 piece " + f"({(~exact).sum()} within the CSV's 1 us rounding of a pose time)") + order = np.argsort(psid, kind="stable") + bounds = np.searchsorted(psid[order], np.arange(psid.max() + 2)) + d_gap = np.full(len(xyz), np.nan, np.float32); d_ctl = np.full(len(xyz), np.nan, np.float32) + for s in range(psid.max() + 1): + win = order[bounds[max(s - 1, 0)]:bounds[min(s + 2, len(bounds) - 1)]] + own = order[bounds[s]:bounds[s + 1]] + g = own[origin[own] == 2]; c = own[control[own]] + base = win[(origin[win] == 0)] + if len(g): + d_gap[g] = cKDTree(xyz[base]).query(xyz[g], workers=-1)[0] + if len(c): + d_ctl[c] = cKDTree(xyz[base[~control[base]]]).query(xyz[c], workers=-1)[0] + gm, cm = origin == 2, control - def pct(x): - q = np.percentile(x, [50, 90, 99]) - return f"median {q[0]:.2f} mm, p90 {q[1]:.2f}, p99 {q[2]:.2f}, max {x.max():.1f} mm" + def row(label, m_g, m_c): + g = d_gap[m_g] * 1000; c = d_ctl[m_c] * 1000 + print(f" {label:>8} {len(g):>10,} {np.median(g):>9.2f} {np.percentile(g, 99):>8.2f} {len(c):>10,} {np.median(c):>9.2f} {np.percentile(c, 99):>8.2f}") - print(f" {len(band):,} check-band points vs {len(odo):,} step 1 points up to {band_r_max:.2f} m") - print(f" all: {pct(d_mm)}") - print(f" with counterpart: {pct(d_mm[~in_gap])} ({(~in_gap).sum():,} points)") - print(f" in chunk gaps: {pct(d_mm[in_gap])} ({in_gap.sum():,} points, no step 1 counterpart)") - print(f" exactly coincident (< 0.15 mm, same trajectory pose as step 1): {100 * (d_mm[~in_gap] < 0.15).mean():.1f} %") - check(np.median(d_mm) < 5.0, f"median NN distance {np.median(d_mm):.2f} mm < 5 mm") + print(f" {'range':>8} {'gap pts':>10} {'median mm':>9} {'p99 mm':>8} {'control':>10} {'median mm':>9} {'p99 mm':>8}") + for lo, hi in ((1, 3), (3, 5), (5, 10), (10, 20), (20, 100)): + row(f"{lo}-{hi} m", gm & (r >= lo) & (r < hi), cm & (r >= lo) & (r < hi)) + row("all", gm, cm) + med_g, med_c = np.median(d_gap[gm]), np.median(d_ctl[cm]) + check(med_g <= 1.2 * med_c + 0.0005, f"gap points as close to step 1's points as a missing slice of step 1's own " + f"(median {med_g * 1000:.2f} vs {med_c * 1000:.2f} mm)") def check_reference(laz, step2_dir): - print("reference (near_added = 0 vs step 2 export)") + print("reference (point_origin = 0 vs step 2 export)") ref_file = step2_dir / "all_step_2.laz" n_ref = laspy.open(ref_file).header.point_count with laspy.open(laz) as f, laspy.open(ref_file) as g: @@ -148,7 +194,7 @@ def check_reference(laz, step2_dir): psid_diff = 0 gps_scale = None for p in f.chunk_iterator(CHUNK): - m = np.asarray(p.near_added) == 0 + m = np.asarray(p.point_origin) == 0 if not m.any(): continue ours = np.column_stack([p.x[m], p.y[m], p.z[m]]) @@ -215,7 +261,8 @@ def main(): print(f" point_source_id {min(stats['psid'])} .. {max(stats['psid'])} ({len(stats['psid'])} scans)") check(1.5e9 < stats["t_min"] < stats["t_max"] < 2.2e9, f"gps_time in Unix seconds ({stats['t_min']:.3f} .. {stats['t_max']:.3f})") check_range(stats, a.near_min) - check_alignment(stats, a.folder) + check_alignment(stats) + check_gaps(laz, a.folder) if a.step2: check_reference(laz, a.step2) check_trajectory(a.folder, a.step2, stats) diff --git a/apps/easy_gui/src/easy_gui_main.cpp b/apps/easy_gui/src/easy_gui_main.cpp index d3f949d2..9b97c7d7 100644 --- a/apps/easy_gui/src/easy_gui_main.cpp +++ b/apps/easy_gui/src/easy_gui_main.cpp @@ -337,11 +337,13 @@ namespace const OutputLayout out(app.output_folder); const ExportStats& e = s.export_stats; ImGui::BulletText( - "%s: %.1f million points (%.1f million from lidar odometry, %.1f million near points), %.0f MB", + "%s: %.1f million points (%.1f million from lidar odometry, %.1f million near and %.1f million gap points " + "added), %.0f MB", out.point_cloud.filename().string().c_str(), - (e.odometry_points + e.near_points + e.check_band_points) / 1e6, + (e.odometry_points + e.near_points + e.gap_points + e.check_band_points) / 1e6, e.odometry_points / 1e6, e.near_points / 1e6, + e.gap_points / 1e6, e.laz_bytes / 1e6); ImGui::BulletText("%s: %zu scanner poses", out.trajectory.filename().string().c_str(), e.trajectory_nodes); ImGui::BulletText("%s describes every field of both files.", out.readme.filename().string().c_str()); @@ -491,10 +493,11 @@ namespace return 1; } std::printf( - "[OK] %llu odometry + %llu near + %llu check-band points; raw read %llu, no-return %llu, outside trajectory %llu; " + "[OK] %llu odometry + %llu near + %llu gap + %llu check-band points; raw read %llu, no-return %llu, outside trajectory %llu; " "range %.3f - %.3f m; %zu poses; LAZ %llu bytes, CSV %llu bytes\n", static_cast(stats.odometry_points), static_cast(stats.near_points), + static_cast(stats.gap_points), static_cast(stats.check_band_points), static_cast(stats.raw_points_read), static_cast(stats.raw_no_return), @@ -547,7 +550,7 @@ int main(int argc, char* argv[]) "usage: %s --headless [--check-band ] [--place-by-raw-time]\n" " %s --export-only [--check-band ] [--place-by-raw-time]\n" "--check-band (testing): also export the raw points between the 1.0 m load filter and , flagged\n" - " near_added = 2, to check that near points line up with the odometry points.\n" + " point_origin = 3, to check that near points line up with the odometry points.\n" "--place-by-raw-time: place all points from the raw files at their real time (export option, default off).\n", argv[0], argv[0]); diff --git a/apps/easy_gui/src/export.cpp b/apps/easy_gui/src/export.cpp index a21bb581..a530fbcf 100644 --- a/apps/easy_gui/src/export.cpp +++ b/apps/easy_gui/src/export.cpp @@ -35,14 +35,14 @@ namespace easy_gui //! Extra bytes of every exported point, in this order. const std::vector kAttributes = { { ExtraBytesAttribute::Type::F32, "range_m", "3D distance from the lidar [m]" }, - { ExtraBytesAttribute::Type::U8, "near_added", "0 odometry, 1 near point added" }, + { ExtraBytesAttribute::Type::U8, "point_origin", "0 step 1, 1 near, 2 gap point" }, }; #pragma pack(push, 1) struct ExtraBytes { float range_m; - uint8_t near_added; + uint8_t point_origin; }; #pragma pack(pop) static_assert(sizeof(ExtraBytes) == 5); @@ -113,6 +113,9 @@ namespace easy_gui double time_s; // lidar clock (Unix epoch seconds on Mandeye) size_t scan; Eigen::Affine3d global; // scan pose (session_poses.mrp) x node pose relative to the scan + //! Last node of a step 1 piece (worker chunk): step 1 takes the points in [first node, last node) of each + //! piece, so raw points from this node until the next piece's first node are not in its output. + bool piece_end; }; void addRange(ExportStats& stats, double range) @@ -215,7 +218,7 @@ namespace easy_gui << "are in metres in a local frame that starts at the scanner's first position (z up, from the IMU).\n\n" << "Files\n-----\n" << " " << laz << " point cloud (LAZ, LAS 1.4, point format 6), " - << formatCount(stats.odometry_points + stats.near_points + stats.check_band_points) << " points\n" + << formatCount(stats.odometry_points + stats.near_points + stats.gap_points + stats.check_band_points) << " points\n" << " " << csv << " trajectory of the scanner (CSV), " << formatCount(stats.trajectory_nodes) << " poses\n" << " easy_gui.log processing log\n" << " work/lio_session/ lidar odometry session; open session.mjs in multi_view_tls_registration_step_2\n" @@ -225,15 +228,12 @@ namespace easy_gui << "clock. They are NOT GPS week time and NOT adjusted standard GPS time, even though the LAS field is called\n" << "gps_time (the LAS header's global encoding bit 0 is 0 accordingly).\n" << (request.place_by_raw_time - ? "Every point carries its raw measurement time (\"place by raw time\" export option): all points were\n" - "taken from the raw recording and placed with the trajectory pose at that time.\n\n" - : "Points from lidar odometry (near_added = 0) carry lidar odometry's time, not the raw measurement time:\n" + ? "Every point carries its raw measurement time (experimental \"place by raw time\" export option).\n\n" + : "Lidar odometry's points (point_origin = 0) carry lidar odometry's time, not the raw measurement time:\n" "within each 0.1 s piece of the recording, odometry spreads its points' times evenly between the first and\n" "last pose of that piece. That is close to the measurement time (typically 1-4 ms off, at most about\n" - "20 ms), but it cannot be used to match points to the raw files. Odometry also places each point with the\n" - "pose at that time, so the same offset shows up as small position differences between odometry points\n" - "and near points: usually a few millimetres, rarely more than 2 cm. Near points (near_added = 1) keep\n" - "their raw measurement time.\n\n") + "20 ms), but it cannot be used to match points to the raw files. Near points and gap points\n" + "(point_origin = 1, 2) carry their raw measurement time.\n\n") << "Point cloud fields (" << laz << ")\n" << std::string(21 + laz.size(), '-') << "\n" << " X, Y, Z position [m], 0.1 mm resolution\n" @@ -245,36 +245,41 @@ namespace easy_gui << " return_number, number_of_returns always 1; classification always 0 (not classified)\n" << " range_m extra bytes, float32: 3D distance from the lidar at the moment of measurement [m];\n" << " can exceed " << outer << " for steep points, as that limit is horizontal\n" - << (request.place_by_raw_time - ? " near_added extra bytes, uint8: 0 = point inside lidar odometry's distance limits, 1 = near point\n" - : " near_added extra bytes, uint8: 0 = point from lidar odometry, 1 = near point added by Easy GUI\n") - << (stats.check_band_points ? " 2 = TEST MODE alignment-check point (duplicate of an odometry point)\n" + << " point_origin extra bytes, uint8: where the point comes from (see below)\n" + << (request.place_by_raw_time ? " 0 = inside lidar odometry's distance limits, placed by raw time\n" + : " 0 = lidar odometry's own point\n") + << " 1 = near point, 2 = gap point (both added by Easy GUI)\n" + << (stats.check_band_points ? " 3 = TEST MODE alignment-check point (duplicate of an odometry point)\n" : "") << "\n" - << "Near points\n-----------\n" + << "Where the points come from\n--------------------------\n" << "Lidar odometry only uses and exports points whose horizontal distance from the lidar (in the lidar's own\n" << "frame) is between " << inner << " and " << outer << " and whose 3D distance is above " << output_filter - << ". Easy GUI adds back\n" - << "the raw " - << "points those limits removed at close range, so the cloud has no hole around the walking path:\n" - << "a raw point is added (near_added = 1) when\n" - << " - its 3D distance from the lidar is at least " << near_min << " (closer returns are mostly the operator's\n" - << " body and the scanner handle; the limit is the advanced setting \"Closest points kept\"), and\n" - << " - lidar odometry left it out because of distance: horizontal distance at most " << inner << ", or 3D\n" - << " distance at most " << output_filter << ", and horizontal distance below " << outer << ", and\n" - << " - it is a real return: points at exactly (0, 0, 0) or with time 0 are \"no return\" records and are\n" - << " never exported.\n" + << ", and it works in\n" + << "0.1 s pieces: it skips the raw points measured after the last pose of a piece and before the first pose\n" + << "of the next one (about 5 % of the recording time). Easy GUI adds both kinds of missing points back from\n" + << "the raw recording, so the cloud has no hole around the walking path and nothing is missing between the\n" + << "pieces.\n\n" + << "point_origin = 0 (" << formatCount(stats.odometry_points) << " points): " + << (request.place_by_raw_time ? "points inside lidar odometry's limits, taken from the raw recording.\n" + : "lidar odometry's own points, as it placed them.\n") + << "point_origin = 1, near points (" << formatCount(stats.near_points) << "): raw points that\n" + << " - are at least " << near_min << " from the lidar in 3D (closer returns are mostly the operator's body and the\n" + << " scanner handle; the limit is the advanced setting \"Closest points kept\"), and\n" + << " - lidar odometry left out because of distance: horizontal distance at most " << inner << ", or 3D\n" + << " distance at most " << output_filter << ", and horizontal distance below " << outer << ".\n" + << "point_origin = 2, gap points (" << formatCount(stats.gap_points) << "): raw points inside lidar odometry's distance\n" + << " limits that it skipped between two 0.1 s pieces.\n" + << "Near and gap points are placed with the trajectory pose lidar odometry uses for points measured at the\n" + << "same time (the last pose before the point), so they line up with lidar odometry's points" + << (request.place_by_raw_time + ? ".\n" + : " to within a few\nmillimetres (rarely more than 2 cm, because of lidar odometry's evenly spread times).\n") << (request.place_by_raw_time - ? "With the \"place by raw time\" option every point, not only the near points, is taken from the raw\n" - "recording and placed with the last trajectory pose before its measurement time, including the\n" - "points lidar odometry skips between its 0.1 s pieces. " - : "Each near point is placed with the trajectory pose lidar odometry uses for points measured at the same\n" - "time (the last pose before the point), so it lines up with the odometry points. ") - << "Points measured where\n" - << "the trajectory has no pose within 10 ms (start and end of the recording) are left out.\n" - << "Near points: " << formatCount(stats.near_points) - << (request.place_by_raw_time ? ", points inside lidar odometry's limits: " : ", odometry points: ") - << formatCount(stats.odometry_points) << ".\n\n" + ? "With the experimental \"place by raw time\" option, point_origin = 0 points are placed the same way.\n" + : "") + << "Points at exactly (0, 0, 0) or with time 0 are \"no return\" records and are never exported, and points\n" + << "measured where the trajectory has no pose within 10 ms (start and end of the recording) are left out.\n\n" << "Trajectory fields (" << csv << ", comma-separated, one header row)\n" << std::string(53 + csv.size(), '-') << "\n" << " timestamp_unix_s time of the pose [Unix epoch seconds], same clock as the point cloud's gps_time\n" @@ -336,6 +341,13 @@ namespace easy_gui stats.filter_threshold_xy_inner = params->filter_threshold_xy_inner; stats.filter_threshold_xy_outer = params->filter_threshold_xy_outer; stats.threshould_output_filter = params->threshould_output_filter; + const int poses_per_piece = params->threshold_nr_poses; + stats.threshold_nr_poses = poses_per_piece; + if (poses_per_piece <= 0) + { + error = "The lidar odometry settings file has no valid threshold_nr_poses."; + return false; + } // --- poses and trajectories exactly as step 2 loads them; the points are streamed below instead progress(0.0, "Reading the lidar odometry result"); @@ -379,8 +391,15 @@ namespace easy_gui error = "The lidar odometry result is incomplete: scan " + std::to_string(i) + " has no trajectory."; return false; } - for (const auto& n : clouds[i].local_trajectory) - nodes.push_back({ n.timestamps.first / 1e9, i, clouds[i].m_pose * n.m_pose }); + // Every step 1 piece has exactly threshold_nr_poses nodes, and save_result concatenates whole pieces into a + // scan, so the piece boundaries are at multiples of it within each scan. + const auto& trajectory = clouds[i].local_trajectory; + for (size_t j = 0; j < trajectory.size(); j++) + nodes.push_back( + { trajectory[j].timestamps.first / 1e9, + i, + clouds[i].m_pose * trajectory[j].m_pose, + static_cast(j % poses_per_piece) == poses_per_piece - 1 }); } std::stable_sort( nodes.begin(), @@ -411,9 +430,10 @@ namespace easy_gui const uint8_t user_data = static_cast(request.lidar_id); const size_t scans_to_copy = request.place_by_raw_time ? 0 : clouds.size(); const double work_units = static_cast(scans_to_copy + request.recording.pairs); - auto writePoint = [&](const Eigen::Vector3d& position, double time_s, uint16_t intensity, size_t scan, double range, NearFlag flag) + auto writePoint = + [&](const Eigen::Vector3d& position, double time_s, uint16_t intensity, size_t scan, double range, PointOrigin origin) { - const ExtraBytes extra{ static_cast(range), flag }; + const ExtraBytes extra{ static_cast(range), origin }; Las14Point p; p.position = position; p.gps_time = time_s; @@ -422,7 +442,10 @@ namespace easy_gui p.user_data = user_data; p.extra_bytes = reinterpret_cast(&extra); addRange(stats, range); - (flag == kFromOdometry ? stats.odometry_points : flag == kNearAdded ? stats.near_points : stats.check_band_points)++; + (origin == kStep1 ? stats.odometry_points + : origin == kNearPoint ? stats.near_points + : origin == kGapPoint ? stats.gap_points + : stats.check_band_points)++; return writer.write(p, error); }; @@ -469,7 +492,7 @@ namespace easy_gui const size_t node = it == times.begin() ? 0 : static_cast(it - times.begin()) - 1; const double range = (lidar_from_sensor * (sensor_from_local[node] * local)).norm(); - if (!writePoint(pc.m_pose * local + offset, rp.gps_time / 1e9, rp.intensity, i, range, kFromOdometry)) + if (!writePoint(pc.m_pose * local + offset, rp.gps_time / 1e9, rp.intensity, i, range, kStep1)) { error = "Could not write the point cloud file (is the disk full?): " + error + "."; return false; @@ -477,7 +500,7 @@ namespace easy_gui } } - // --- 2. near points (with place_by_raw_time: all points) from the raw recording, one file at a time + // --- 2. near and gap points (with place_by_raw_time: all points) from the raw recording, one file at a time const double inner = params->filter_threshold_xy_inner; const double outer = params->filter_threshold_xy_outer; const double output_filter = params->threshould_output_filter; @@ -487,7 +510,7 @@ namespace easy_gui return false; progress( static_cast(scans_to_copy + f) / work_units, - std::string(request.place_by_raw_time ? "Writing the point cloud" : "Adding the near points") + " (file " + + std::string(request.place_by_raw_time ? "Writing the point cloud" : "Adding the near and gap points") + " (file " + std::to_string(f + 1) + " of " + std::to_string(request.recording.pairs) + ")"); const fs::path& raw_file = request.recording.laz_files[f]; @@ -519,30 +542,37 @@ namespace easy_gui // Step 1's distance filters: load_point_cloud keeps inner < range_xy < outer, points_to_vector keeps // |sensor| > threshould_output_filter. Near points are exactly the complement, at close range. const bool step1_keeps = range_xy > inner && range_xy < outer && sensor.norm() > output_filter; - NearFlag flag; - if (step1_keeps) - { - if (request.place_by_raw_time) - flag = kFromOdometry; - else if (range_xy < request.check_band_outer_m) - flag = kCheckBand; - else - continue; // exported from scan_lio_N.laz in pass 1 - } - else if (range >= request.near_min_range_m && range_xy < outer) - flag = kNearAdded; - else + if (!step1_keeps && (range < request.near_min_range_m || range_xy >= outer)) continue; const double t = rp.gps_time; const auto it = std::lower_bound(node_times.begin(), node_times.end(), t); if (it == node_times.begin() || t - *(it - 1) > kNodeTolerance_s) { - stats.raw_outside_trajectory++; + if (!step1_keeps || request.place_by_raw_time) + stats.raw_outside_trajectory++; continue; } + // Placed with the last node before the point (step 1's rule). Whether step 1 has the point depends on + // the last node at or before it: a piece's last node starts a gap step 1 skips. const Node& node = nodes[static_cast(it - node_times.begin()) - 1]; - if (!writePoint(node.global * sensor + offset, t, rp.intensity, node.scan, range, flag)) + const size_t at_or_before = + static_cast(std::upper_bound(node_times.begin(), node_times.end(), t) - node_times.begin()) - 1; + const bool in_gap = nodes[at_or_before].piece_end; + + PointOrigin origin; + if (!step1_keeps) + origin = kNearPoint; + else if (in_gap) + origin = kGapPoint; + else if (request.place_by_raw_time) + origin = kStep1; + else if (range_xy < request.check_band_outer_m) + origin = kCheckBand; + else + continue; // step 1 has this point: exported from scan_lio_N.laz in pass 1 + + if (!writePoint(node.global * sensor + offset, t, rp.intensity, node.scan, range, origin)) { error = "Could not write the point cloud file (is the disk full?): " + error + "."; return false; diff --git a/apps/easy_gui/src/export.h b/apps/easy_gui/src/export.h index 564d8c3e..d72854be 100644 --- a/apps/easy_gui/src/export.h +++ b/apps/easy_gui/src/export.h @@ -11,12 +11,13 @@ namespace easy_gui { - //! Values of near_added, the extra-bytes flag of every exported point. - enum NearFlag : uint8_t + //! Values of point_origin, the extra-bytes field of every exported point. + enum PointOrigin : uint8_t { - kFromOdometry = 0, // exported by lidar odometry (step 1) - kNearAdded = 1, // raw point step 1's distance filters removed, added by Easy GUI - kCheckBand = 2, // test mode only: raw point step 1 also exported, added again to check the alignment + kStep1 = 0, // lidar odometry's (step 1's) own point, placed by step 1 + kNearPoint = 1, // raw point step 1's distance filters removed, added by Easy GUI at its real time + kGapPoint = 2, // raw point inside step 1's limits that step 1 skipped between its 0.1 s pieces, added at its real time + kCheckBand = 3, // test mode only: raw point step 1 also exported, added again to check the alignment }; struct ExportRequest @@ -26,12 +27,13 @@ namespace easy_gui int lidar_id = 0; // user_data of the raw points (single-lidar recordings only) double near_min_range_m = 0.4; // near points closer than this (3D) are left out //! Test mode: > 0 also adds the raw points with filter_threshold_xy_inner < horizontal range < this, which step 1 - //! exported too, flagged kCheckBand, so their placement can be compared with step 1's own points. + //! exported too, as kCheckBand, so their placement can be compared with step 1's own points. double check_band_outer_m = 0.0; - //! Place every point from the raw files with the trajectory pose at its real gps_time, instead of taking step - //! 1's points (scan_lio_N.laz, placed at step 1's ramp time). Same distance filters and near-point rule; raw - //! gps_time is written; includes the raw points step 1 drops between its 0.1 s pieces. near_added = 0 then - //! means "inside step 1's distance limits". check_band_outer_m is ignored (those points are all included). + //! Experimental: place every point from the raw files with the trajectory pose at its real gps_time, instead of + //! taking step 1's points (scan_lio_N.laz, placed at step 1's ramp time). Same distance filters, near-point and + //! gap-point rules; raw gps_time is written. point_origin = 0 then means "inside step 1's distance limits, and + //! not in a gap". check_band_outer_m is ignored (those points are all included). Measured slightly less sharp + //! than the default (see CLAUDE.md), hence off. bool place_by_raw_time = false; std::filesystem::path laz_file; std::filesystem::path trajectory_file; @@ -42,9 +44,10 @@ namespace easy_gui { size_t scans = 0; size_t trajectory_nodes = 0; - uint64_t odometry_points = 0; // near_added = 0 - uint64_t near_points = 0; // near_added = 1 - uint64_t check_band_points = 0; // near_added = 2 + uint64_t odometry_points = 0; // point_origin = 0 + uint64_t near_points = 0; // point_origin = 1 + uint64_t gap_points = 0; // point_origin = 2 + uint64_t check_band_points = 0; // point_origin = 3 uint64_t raw_points_read = 0; uint64_t raw_no_return = 0; // (0,0,0) or gps_time 0: never exported uint64_t raw_outside_trajectory = 0; // exported candidates with no trajectory node within 10 ms @@ -57,6 +60,7 @@ namespace easy_gui double filter_threshold_xy_inner = 0.0; double filter_threshold_xy_outer = 0.0; double threshould_output_filter = 0.0; + int threshold_nr_poses = 0; // trajectory poses per step 1 piece (worker chunk) }; //! Called with the fraction done (0..1) and a short description of the current step. diff --git a/apps/easy_gui/src/las14_writer.cpp b/apps/easy_gui/src/las14_writer.cpp index 264a51b4..7e1c42dd 100644 --- a/apps/easy_gui/src/las14_writer.cpp +++ b/apps/easy_gui/src/las14_writer.cpp @@ -11,7 +11,7 @@ namespace easy_gui namespace { //! LAS 1.4 R15 "Extra Bytes" descriptor. Written by hand rather than with laszip_add_attribute(), which always - //! sets the "scale" option bit: readers then scale the values, so laspy would return the uint8 near_added flag as + //! sets the "scale" option bit: readers then scale the values, so laspy would return the uint8 point_origin field as //! float64. #pragma pack(push, 1) struct ExtraBytesDescriptor diff --git a/apps/easy_gui/src/pipeline.cpp b/apps/easy_gui/src/pipeline.cpp index 618785dc..9093b9ba 100644 --- a/apps/easy_gui/src/pipeline.cpp +++ b/apps/easy_gui/src/pipeline.cpp @@ -373,7 +373,8 @@ namespace easy_gui { std::ostringstream s; - s << "[easy_gui] export: " << stats.odometry_points << " odometry points + " << stats.near_points << " near points"; + s << "[easy_gui] export: " << stats.odometry_points << " odometry points + " << stats.near_points << " near points + " + << stats.gap_points << " gap points"; if (stats.check_band_points) s << " + " << stats.check_band_points << " check-band points"; s << " (" << stats.raw_no_return << " of " << stats.raw_points_read << " raw points were no-return, " diff --git a/apps/easy_gui/src/preflight.cpp b/apps/easy_gui/src/preflight.cpp index 1ea70974..c67b66a7 100644 --- a/apps/easy_gui/src/preflight.cpp +++ b/apps/easy_gui/src/preflight.cpp @@ -243,8 +243,8 @@ namespace easy_gui uint64_t estimateResultBytes(uint64_t raw_laz_bytes) { - // Measured on a 117 M-point forest walk (734 MB raw LAZ): session 0.82x + point cloud 1.24x + trajectory CSV - // 0.03x = 2.08x; 2.25x leaves some margin (the near-point share, and so the LAZ size, depends on the site). + // Measured on a 117 M-point forest walk (734 MB raw LAZ): session 0.82x + point cloud 1.29x (with near and gap + // points) + trajectory CSV 0.03x = 2.14x; 2.25x leaves some margin (the near-point share depends on the site). return raw_laz_bytes * 9 / 4; } diff --git a/apps/easy_gui/src/preflight.h b/apps/easy_gui/src/preflight.h index 03d3e979..78805bb9 100644 --- a/apps/easy_gui/src/preflight.h +++ b/apps/easy_gui/src/preflight.h @@ -67,7 +67,7 @@ namespace easy_gui uint64_t estimateWorkBytes(uint64_t total_points); //! Disk space of what is kept: the step 1 session (scan_lio LAZ, 0.82x the raw LAZ size measured), the exported - //! LAZ (1.4 PF6 with extra bytes and the near points, 1.24x measured) and the trajectory CSV (0.03x): 2.25x. + //! LAZ (1.4 PF6 with extra bytes, near and gap points, 1.29x measured) and the trajectory CSV (0.03x): 2.25x. uint64_t estimateResultBytes(uint64_t raw_laz_bytes); //! True if `inner` is `outer` or lies inside it (both made absolute and normalised first). diff --git a/apps/easy_gui/tests/test_export.cpp b/apps/easy_gui/tests/test_export.cpp index f227de44..01eaf809 100644 --- a/apps/easy_gui/tests/test_export.cpp +++ b/apps/easy_gui/tests/test_export.cpp @@ -29,7 +29,7 @@ namespace uint16_t point_source_id; uint8_t user_data; float range_m; - uint8_t near_added; + uint8_t point_origin; }; struct ReadFile @@ -80,7 +80,7 @@ namespace if (p->num_extra_bytes >= 5) { std::memcpy(&q.range_m, p->extra_bytes, 4); - q.near_added = p->extra_bytes[4]; + q.point_origin = p->extra_bytes[4]; } r.points.push_back(q); } @@ -249,7 +249,7 @@ TEST_CASE("Las14Writer writes LAS 1.4 point format 6 with named, unscaled extra std::string error; REQUIRE(writer.open( file.string(), - { { ExtraBytesAttribute::Type::F32, "range_m", "range" }, { ExtraBytesAttribute::Type::U8, "near_added", "flag" } }, + { { ExtraBytesAttribute::Type::F32, "range_m", "range" }, { ExtraBytesAttribute::Type::U8, "point_origin", "flag" } }, Eigen::Vector3d(100.0, 200.0, 0.0), 0.0001, "test", @@ -287,7 +287,7 @@ TEST_CASE("Las14Writer writes LAS 1.4 point format 6 with named, unscaled extra CHECK(r.header.max_z == doctest::Approx(5.0)); REQUIRE(r.extra_names.size() == 2); CHECK(r.extra_names[0] == "range_m"); - CHECK(r.extra_names[1] == "near_added"); + CHECK(r.extra_names[1] == "point_origin"); CHECK(r.extra_types[0] == 9); // float CHECK(r.extra_types[1] == 1); // unsigned char CHECK(r.extra_options[0] == 0); // no scale: readers return the raw float / uint8 @@ -299,9 +299,9 @@ TEST_CASE("Las14Writer writes LAS 1.4 point format 6 with named, unscaled extra CHECK(r.points[0].point_source_id == 4321); CHECK(r.points[0].user_data == 2); CHECK(r.points[0].range_m == 12.5f); - CHECK(r.points[0].near_added == 1); + CHECK(r.points[0].point_origin == 1); CHECK(r.points[1].range_m == 0.0f); - CHECK(r.points[1].near_added == 0); + CHECK(r.points[1].point_origin == 0); } TEST_CASE("exportSession: step 1 points as step 2 places them, near points with step 1's transform chain") @@ -316,6 +316,11 @@ TEST_CASE("exportSession: step 1 points as step 2 places them, near points with const Eigen::Vector3d near_a(0.5, 0.3, 0.2); // horizontal 0.58 m, 3D 0.62 m: near point const Eigen::Vector3d near_steep(0.6, 0.0, 1.5); // horizontal 0.6 m but 3D 1.62 m: still a near point const Eigen::Vector3d band(1.1, 0.0, 0.0); // step 1 exports this one itself + // step 1 has 20 nodes per piece (threshold_nr_poses) and skips [last node of a piece, next piece's first node) + const Eigen::Vector3d gap_a(2.5, 1.0, 0.3); + const Eigen::Vector3d gap_b(-4.0, 2.0, 1.0); + const double t_gap_a = S::nodeTime(19) + 0.002; + const double t_gap_b = S::nodeTime(39) + 0.003; std::vector file0 = { { near_a, t5, 11 }, { Eigen::Vector3d(0.2, 0.1, 0.0), t5 }, // 3D 0.22 m < near_min_range @@ -324,6 +329,9 @@ TEST_CASE("exportSession: step 1 points as step 2 places them, near points with { Eigen::Vector3d(3.0, 0.0, 0.0), t5 }, // step 1's point { band, t5 }, { near_a, S::kT0 - 1.0 }, // before the trajectory: no pose within 10 ms + { gap_a, t_gap_a, 13 }, // inside step 1's limits, but in the gap after piece 0 (nodes 0-19): gap point + { gap_b, t_gap_b }, // gap at the end of scan 0 (node 39 ends piece 1), before scan 1's first node + { Eigen::Vector3d(2.0, 0.5, 0.1), S::nodeTime(18) + 0.002 }, // inside a piece: step 1's point, not added }; std::vector file1 = { { near_steep, t47, 12 }, @@ -365,7 +373,9 @@ TEST_CASE("exportSession: step 1 points as step 2 places them, near points with CHECK(stats.trajectory_nodes == 2 * S::kNodes); CHECK(stats.odometry_points == s.odometry_points); CHECK(stats.near_points == 2); + CHECK(stats.gap_points == 2); CHECK(stats.check_band_points == 0); + CHECK(stats.threshold_nr_poses == 20); CHECK(stats.raw_points_read == file0.size() + file1.size()); CHECK(stats.raw_no_return == 2); CHECK(stats.raw_outside_trajectory == 2); @@ -373,7 +383,7 @@ TEST_CASE("exportSession: step 1 points as step 2 places them, near points with CHECK(stats.threshould_output_filter == 1.0); // Easy GUI's output setting, read back from the session const ReadFile r = readLaz(request.laz_file); - REQUIRE(r.points.size() == s.odometry_points + 2); + REQUIRE(r.points.size() == s.odometry_points + 4); // raw file order: near_a, gap_a, gap_b, near_steep // odometry points: scan pose x local point, range from the sensor frame, ramp time in seconds for (size_t i = 0; i < s.odometry_points; i++) @@ -381,7 +391,7 @@ TEST_CASE("exportSession: step 1 points as step 2 places them, near points with const auto& p = r.points[i]; const int scan = static_cast(i / (2 * S::kNodes)); const int node = scan * S::kNodes + static_cast(i % (2 * S::kNodes)) / 2; - CHECK(p.near_added == kFromOdometry); + CHECK(p.point_origin == kStep1); CHECK(p.point_source_id == scan); CHECK(p.user_data == S::kLidarId); CHECK(near(p.xyz, S::pose(node) * s.sensor_points[i])); @@ -391,14 +401,26 @@ TEST_CASE("exportSession: step 1 points as step 2 places them, near points with // near points: pose of the last node before the raw time, raw time and intensity kept const auto& a = r.points[s.odometry_points]; - CHECK(a.near_added == kNearAdded); + CHECK(a.point_origin == kNearPoint); CHECK(near(a.xyz, S::pose(5) * near_a)); CHECK(a.gps_time == t5); CHECK(a.intensity == 11); CHECK(a.point_source_id == 0); CHECK(a.range_m == doctest::Approx(near_a.norm())); - const auto& b = r.points[s.odometry_points + 1]; - CHECK(b.near_added == kNearAdded); + // gap points: same placement rule, raw time, scan of the node + const auto& g = r.points[s.odometry_points + 1]; + CHECK(g.point_origin == kGapPoint); + CHECK(near(g.xyz, S::pose(19) * gap_a)); + CHECK(g.gps_time == t_gap_a); + CHECK(g.intensity == 13); + CHECK(g.point_source_id == 0); + CHECK(g.range_m == doctest::Approx(gap_a.norm())); + const auto& g2 = r.points[s.odometry_points + 2]; + CHECK(g2.point_origin == kGapPoint); + CHECK(near(g2.xyz, S::pose(39) * gap_b)); + CHECK(g2.point_source_id == 0); + const auto& b = r.points[s.odometry_points + 3]; + CHECK(b.point_origin == kNearPoint); CHECK(near(b.xyz, S::pose(47) * near_steep)); CHECK(b.point_source_id == 1); CHECK(b.range_m == doctest::Approx(near_steep.norm())); @@ -434,7 +456,8 @@ TEST_CASE("exportSession: step 1 points as step 2 places them, near points with const std::string text((std::istreambuf_iterator(readme)), std::istreambuf_iterator()); CHECK(text.find("Unix epoch seconds") != std::string::npos); CHECK(text.find("at least 0.40 m") != std::string::npos); - CHECK(text.find("near_added") != std::string::npos); + CHECK(text.find("point_origin") != std::string::npos); + CHECK(text.find("gap points (2)") != std::string::npos); SUBCASE("check-band test mode adds the 1.0-1.2 m raw points with flag 2") { @@ -450,9 +473,10 @@ TEST_CASE("exportSession: step 1 points as step 2 places them, near points with CHECK(stats.near_points == 2); CHECK(stats.check_band_points == 1); const ReadFile rb = readLaz(request.laz_file); - REQUIRE(rb.points.size() == s.odometry_points + 3); - const auto& c = rb.points[s.odometry_points + 1]; // raw file order: near_a, band, near_steep - CHECK(c.near_added == kCheckBand); + CHECK(stats.gap_points == 2); + REQUIRE(rb.points.size() == s.odometry_points + 5); + const auto& c = rb.points[s.odometry_points + 1]; // raw file order: near_a, band, gap_a, gap_b, near_steep + CHECK(c.point_origin == kCheckBand); CHECK(near(c.xyz, S::pose(5) * band)); CHECK(c.range_m == doctest::Approx(band.norm())); } @@ -469,21 +493,25 @@ TEST_CASE("exportSession: step 1 points as step 2 places them, near points with }, stats, error)); - CHECK(stats.odometry_points == 2); // the 3 m point and the 1.1 m point, not scan_lio's points + CHECK(stats.odometry_points == 3); // the 3 m, 1.1 m and node-18 points, not scan_lio's points CHECK(stats.near_points == 2); + CHECK(stats.gap_points == 2); CHECK(stats.check_band_points == 0); CHECK(stats.raw_outside_trajectory == 2); const ReadFile rr = readLaz(request.laz_file); - REQUIRE(rr.points.size() == 4); // raw file order: near_a, 3 m, band, near_steep - CHECK(rr.points[0].near_added == kNearAdded); - CHECK(rr.points[1].near_added == kFromOdometry); + REQUIRE(rr.points.size() == 7); // raw file order: near_a, 3 m, band, gap_a, gap_b, node 18, near_steep + CHECK(rr.points[0].point_origin == kNearPoint); + CHECK(rr.points[1].point_origin == kStep1); CHECK(near(rr.points[1].xyz, S::pose(5) * Eigen::Vector3d(3.0, 0.0, 0.0))); CHECK(rr.points[1].gps_time == t5); CHECK(rr.points[1].range_m == doctest::Approx(3.0)); - CHECK(rr.points[2].near_added == kFromOdometry); + CHECK(rr.points[2].point_origin == kStep1); CHECK(near(rr.points[2].xyz, S::pose(5) * band)); - CHECK(rr.points[3].near_added == kNearAdded); - CHECK(rr.points[3].point_source_id == 1); + CHECK(rr.points[3].point_origin == kGapPoint); + CHECK(rr.points[5].point_origin == kStep1); + CHECK(near(rr.points[5].xyz, S::pose(18) * Eigen::Vector3d(2.0, 0.5, 0.1))); + CHECK(rr.points[6].point_origin == kNearPoint); + CHECK(rr.points[6].point_source_id == 1); } SUBCASE("cancel stops the export") diff --git a/apps/easy_gui/tests/test_pipeline_integration.cpp b/apps/easy_gui/tests/test_pipeline_integration.cpp index f1db6315..7eb0c513 100644 --- a/apps/easy_gui/tests/test_pipeline_integration.cpp +++ b/apps/easy_gui/tests/test_pipeline_integration.cpp @@ -120,6 +120,7 @@ TEST_CASE("a real run ends with the point cloud, the trajectory and the README" CHECK(fs::exists(layout.readme)); CHECK(s.export_stats.odometry_points > 0); CHECK(s.export_stats.near_points > 0); + CHECK(s.export_stats.gap_points > 0); CHECK(s.export_stats.check_band_points == 0); CHECK(s.export_stats.range_min_m >= Settings{}.near_min_range_m); CHECK(fs::exists(layout.session / "session.mjs")); // kept for step 2 From c90c46dbd3114d4e1ff05caf37faebb73443e35e Mon Sep 17 00:00:00 2001 From: Martin Mokros Date: Sun, 18 Oct 2026 11:27:20 +0100 Subject: [PATCH 05/23] Easy GUI: move design notes to apps/easy_gui/docs/DESIGN.md; stop tracking CLAUDE.md DESIGN.md collects the design decisions, Mandeye raw-data facts, what step 1 does to points, the export, validation and sharpness results, written for collaborators and reviewers. CLAUDE.md becomes local working notes (untracked). Source comments now point to DESIGN.md. Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 149 ----------- apps/easy_gui/docs/DESIGN.md | 399 ++++++++++++++++++++++++++++ apps/easy_gui/src/easy_gui_main.cpp | 2 +- apps/easy_gui/src/export.h | 2 +- apps/easy_gui/src/forestry_preset.h | 2 +- 5 files changed, 402 insertions(+), 152 deletions(-) delete mode 100644 CLAUDE.md create mode 100644 apps/easy_gui/docs/DESIGN.md diff --git a/CLAUDE.md b/CLAUDE.md deleted file mode 100644 index 336fffe2..00000000 --- a/CLAUDE.md +++ /dev/null @@ -1,149 +0,0 @@ -# CLAUDE.md - -This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. - -## Project - -HDMapping: C++20 desktop apps that turn raw Mandeye (handheld LiDAR + IMU) recordings into registered, georeferenced LAZ point clouds. Optimization is implemented from scratch with Eigen (Jacobians generated by `3rdparty/observation_equations`), with no Ceres, g2o or GTSAM. User-facing docs live in the GitHub wiki (https://github.com/MapsHD/HDMapping/wiki). - -## Build, test, format - -Dependencies are git submodules: `git submodule update --init --recursive` (or `python3 scripts/run_submodules_sync_update.py`). CMake >= 4.0 is required. raylib 5.5 and rlImGui are fetched at configure time via `FetchContent`, so the first configure needs network access. - -```bash -# Configure (same flags as CI; on macOS CI also passes -DFREEGLUT_COCOA=ON) -cmake -B build -DCMAKE_BUILD_TYPE=Release -DBUILD_WITH_UTILITY_APPLICATION=ON -DBUILD_TESTING=ON -cmake --build build --config Release -j - -# Build a single app -cmake --build build --target multi_view_tls_registration_step_2 - -# Run all tests (from the build dir) -cd build && ctest -C Release --output-on-failure -# Run one test suite: hdmapping_shared_tests, calib_core_tests, lidar_odometry_step_1_tests, hdmapping_rosbags_tests -ctest -R calib_core_tests --output-on-failure -# Run one doctest case directly -./build/bin/calib_core_tests --test-case="" - -# Format (CI enforces clang-format 21 via scripts/run_check_clang_format.py) -python3 scripts/run_clang_format.py -python3 scripts/run_check_clang_format.py -``` - -Binaries go to `build/bin`. Optional CMake switches: -- `BUILD_WITH_CLI_TOOLS`: `apps/console_tools` converters. -- `BUILD_WITH_HD_MAPPER_APPLICATION`: `hd_mapper`. -- `BUILD_WITH_UTILITY_APPLICATION`: calibration, forestry and Livox tools. -- `BUILD_WITH_PYBIND`: `pybind/` Python bindings. -- `HDMAPPING_PROFILER=NONE|UTL|TRACY`: profiler backend. -- `HD_CPU_OPTIMIZATION`: CPU optimization target. - -Tests use vendored doctest (`3rdparty/doctest`) and are only added when `BUILD_TESTING=ON`. A new test directory must also be registered in the root `CMakeLists.txt`. - -## Pipeline (see images/workflow.png) - -1. `mandeye_raw_data_viewer`: QC of the raw recording (`lidar*.laz`, `imu*.csv`, `*.sn`). -2. `lidar_odometry_step_1`: LiDAR-inertial odometry. `save_result()` in `apps/lidar_odometry_step_1/lidar_odometry.cpp` writes a session directory (by default the next free `/lio_result_N`): `session.mjs` (JSON), `scan_lio_N.laz`, `trajectory_lio_N.csv`, `session_poses.mrp` / `session_ini_poses.mri`, `point_sizes_per_chunk.json`, and the parameters TOML. Parameters are stored as TOML (`toml_io.cpp`). -3. `mandeye_single_session_viewer`: QC of the odometry result. -4. `multi_view_tls_registration_step_2`: refines one session (pose-graph loop closure, ICP/NDT, GNSS/GCP/control-point georeferencing) and exports the result. It also produces ground-truth sessions. -5. `multi_session_registration_step_3`: aligns several sessions using a project JSON (`session_file_names`) and `multi_session_factor_graph.cpp`, then writes poses back into each session. `multi_session_registration_no_gui.cpp` is the headless variant. - -The file that ties the steps together is the session JSON (`session.mjs` from step 1, `*.json` also accepted), loaded and saved by `Session` (`core/include/Core/session.h`). `Session::load` also accepts a single `.laz`/`.las` file (`load_from_point_cloud`). The CMake target names (`*_step_1/2/3`) differ from the app directory names. - -The step-1 sources (`lidar_odometry*.cpp`, `raw_cloud_loader.cpp`, `toml_io.cpp`, ...) are not a library. They are listed by relative path in the CMakeLists of the raw viewer, the single-session viewer and step 2. Changing their signatures affects all of those apps. - -## Libraries - -- `core/` builds several targets (see `core/CMakeLists.txt`): - - `core_math`: registration and optimization code, always `WITH_GUI=0`. Its generated Jacobian headers are slow to compile. - - `core` (`WITH_GUI=1`, adds the ImGui manual loop-closure and observation-picking panels) and `core_no_gui` (only built for tests, pybind and CLI tools). - - `core_pfd` (file dialogs), `core_e57` and `core_raylib` (shared raylib rendering). - - Classes shared between GUI and non-GUI builds (e.g. `Session`) must not change their layout under `WITH_GUI`. -- `calib_core/`: cameraโ€“LiDAR calibration library for the `camera_lidar_*` apps. -- `raylib_widgets/`: shared UI for the raylib apps. `AppShell` provides the dockspace, the Console dock and the info/shortcuts window; other parts are `OrbitCamera`, `PointPicking` and `PointBufferParts`. -- `shared/include/HDMapping/`: header-only helpers (`HDMAPPING_ASSERT`, pose interpolation). -- `rosbags/`: MCAP read/write. - -## GUI architecture (two stacks, mid-migration) - -All GUIs are immediate-mode Dear ImGui (vendored `3rdparty/imgui`, docking branch) with ImGuizmo and implot. - -- **Legacy: freeglut + OpenGL2.** Used by step 1, the raw and single-session viewers, the `*_legacy` apps and most utilities. These link the `imgui` target from `cmake/imgui.cmake`, which has the GLUT and OpenGL2 backends compiled in. GLUT callbacks drive `glutMainLoop()`, and the 3D view is fixed-function GL. -- **New: raylib + rlImGui.** Used by step 2, step 3, `session_uncertainty_calculation`, `session_to_session_rigid_registration` and the `camera_lidar_*` apps. - - `cmake/raylib.cmake` builds separate `imgui_raylib` and `imguizmo_raylib` targets from the same vendored sources, without any backend. Never link both `imgui` and `imgui_raylib` into one binary. - - Main loop pattern: `while (!WindowShouldClose()) { rlImGuiBegin(); ShowMainDockSpace(); ...; rlImGuiEnd(); }`, with drag and drop via `IsFileDropped()`. - - `IMGUI_DISABLE_OBSOLETE_FUNCTIONS` is deliberately not defined, because `core`'s GUI sources call obsolete ImGui APIs. -- The `*_legacy` directories are near-copies of the GLUT versions of steps 2 and 3, kept while the raylib ports mature. A fix to shared logic may need to be applied in both copies. - -## Conventions - -- clang-format (`.clang-format`) is checked in CI; run `scripts/run_clang_format.py` before committing. -- Comments in this codebase explain *why*, often with `NOTE(author)` / `TODO(author)` tags. CMake files carry detailed rationale comments for target splits; read them before restructuring targets. - -## Easy GUI (branch `easy-gui`) - -Goal: one app: raw Mandeye folder โ†’ step 1 (odometry, forestry preset) โ†’ step 2 refinement โ†’ export. No step 3. Outputs: (1) LAZ point cloud with all available per-point attributes, (2) trajectory CSV with timestamp, x, y, z and all other available fields. - -Constraints: raylib stack; reuse existing code, don't duplicate algorithms; don't modify `core_math`; existing apps stay unchanged; progress display and plain-language errors. - -Decisions: -- v1 = odometry โ†’ export, no refinement. Validate it against a manual step 1 โ†’ step 2 run first. v2 adds gated pose-graph loop closure (`PoseGraphLoopClosure`, auto-detected candidates, edges rejected if ICP fails) behind a toggle. Do not use `PoseGraphSLAM::optimize`. -- Output LAZ: LAS 1.4, point format 6, extra-bytes attributes. `gps_time` = Unix epoch seconds; global encoding bit 0 (GPS time type) = 0, since it is not adjusted GPS time. Document the time base. -- Default export = hybrid: step 1's own points as step 1 placed them (`point_origin = 0`), plus near points (1) and gap points (2), both from the raw files and placed by real gps_time. Gap points = raw points inside step 1's distance limits that step 1 skipped between its 0.1 s pieces: the last trajectory node at or before t is a piece's last node (`j % threshold_nr_poses == threshold_nr_poses โˆ’ 1` within the scan; step 1 takes [first, last node) of each piece). `point_origin = 3` = check-band test points. Placing step 1's own points by raw time is only an experimental option (`place_by_raw_time`), because it measured slightly less sharp (see Export). -- Multi-lidar rigs are out of scope for v1; the pre-flight check rejects recordings with more than one lidar id. -- Near points (added by Easy GUI at export, never seen by odometry): keep raw points with `near_min_range` โ‰ค 3D range **and** that step 1's distance filters removed. As implemented, that is the exact complement: horizontal range โ‰ค `filter_threshold_xy_inner` (1.0 m) or calibrated 3D norm โ‰ค `threshould_output_filter`, and horizontal range < `filter_threshold_xy_outer`. Always drop exact (0,0,0) and gps_time 0 no-return points (~15 % of raw points). Transform each one with step 1's rule at its *raw* gps_time: the last trajectory node with t_node < t, if within 10 ms (step 1's own tolerance), so `global = mrp[scan] ยท csv_pose[node] ยท calib ยท p_raw`. Mark it `point_origin = 1` (extra bytes, uint8; `user_data` keeps the lidar id). `near_min_range` is an advanced setting, default 0.4 m: measured raw data shows the operator's body/handle only as small blobs inside 0.4 m, and near points beyond that are environment in all directions. -- For the near rule to meet step 1's output without a gap, Easy GUI writes `threshould_output_filter = filter_threshold_xy_inner` (1.0 instead of the preset's 1.5). It is output-only, so the trajectory is unchanged; points with horizontal range โ‰ฅ 1.0 m then all come from step 1, and points below that come from the near rule. -- Easy GUI never writes into the recording folder (it may be on the Mandeye USB stick). `load_data` puts `cache/` and `preview/` next to the input files, so step 1 runs on a staging folder of symlinks inside the Easy GUI output folder. -- Forestry preset = `LidarOdometryParams` defaults + the 18 overrides in `lidar_odometry_gui.cpp` "Presets โ†’ 2 โ€ฆ precise forestry". There is no shared preset function; the values are inline GUI code (also copied in the precision_forestry app). - -### Raw Mandeye data (verified on a real file + firmware source) -- LAZ: LAS 1.2 PF1. Real fields: X,Y,Z (sensor frame), intensity (0โ€“160), gps_time, user_data = lidar id. No extra bytes; other standard fields are 0. -- gps_time = Unix epoch seconds (the header wrongly flags adjusted GPS time). -- classification is constant 20 and meaningless: the firmware never copies the Livox tag. A planned firmware fix will put the tag there (bits 0โ€“1 adhesion, 2โ€“3 particles, 4โ€“5 other). -- IMU csv: timestamp (ns, same clock as gps_time), gyroXYZ, accXYZ (g), imuId, timestampUnix (ms, host clock โ€” ignore). 200 Hz. - -### Code (`apps/easy_gui`) -- `easy_gui_core` (static lib, no raylib): `preflight` (file classification mirroring load_data, LAZ/IMU/calibration/disk checks), `staging`, `child_process` (POSIX posix_spawn + process group; Windows CreateProcess + job object, not yet compiled on Windows), `progress_parser` (step 1 log โ†’ phases), `forestry_preset` (writes the step 1 TOML via TomlIO), `export` (M2, see below), `las14_writer` (streaming LAS 1.4 PF6 LAZ writer with a hand-written extra-bytes VLR), `pipeline` (worker thread). Compiles `lidar_odometry_utils.cpp` + `toml_io.cpp` by path with `WITH_GUI=0`; links `core_no_gui` (for `Session`) + `core_math`, not `core`. `core_no_gui` is therefore always built now (core/CMakeLists.txt used to build it only with tests/pybind/CLI tools). -- `easy_gui` (raylib + rlImGui window); `easy_gui --headless ` runs the same pipeline without a window (exit 0/1/2); `easy_gui --export-only ` runs only the export on an existing session (~1.5 min instead of ~22); both take `--check-band ` (test mode, see Export); `--screenshot [recording]` saves the window after ~2 s (raylib saves into the working directory). -- Building `easy_gui` also builds `lidar_odometry_step_1` (`add_dependencies`), which is found next to the executable under its plain or `HDMAPPING_BINARY_PREFIX` name; `HDMAPPING_STEP1_EXECUTABLE` overrides the lookup. -- Output folder: `point_cloud.laz`, `trajectory.csv`, `README.txt` (field/time-base/near-rule description generated with the run's values), `easy_gui.log`, `work/easy_gui_step1.toml`, `work/lio_session/` (kept), `work/staging/` (removed after success unless "keep intermediate files"; all of `work/` is removed after failure or cancel, except that an export failure keeps `work/lio_session/`, since odometry took most of the run). -- Tests: `easy_gui_tests` (doctest). Integration tests against the real step 1 binary run only with `EASY_GUI_TEST_RECORDING=` (e.g. `~/data/mandeye_test/mini4`, symlinks to the first 4 pairs). -- Disk: step 1 needs ~108 B per raw point at peak (cache 7.5 GB + chunk caches ~4 GB + scan_lio for 117 M points). The pre-flight check estimates 120 B per raw point of temporary space + 2.25ร— the raw LAZ size for results (measured: session 0.82ร— + point_cloud.laz 1.29ร— with near and gap points + trajectory.csv 0.03ร— = 2.14ร—), always reports the breakdown, warns below 1.5ร— and refuses below 1ร—. -- Measured on the test recording (117 M raw points, 115 pairs, 10-min walk): step 1 takes ~21 min, peak memory footprint 8.5 GB. Easy GUI (output filter 1.0) vs manual preset 2 (1.5): trajectory **byte-identical** (`session_poses.mrp`, `trajectory_lio_*.csv`), so step 1 is deterministic here and the output filter is confirmed output-only; scan_lio points 89.33 M vs 84.21 M (+5.11 M, +6.1 %). Runs are in `~/data/mandeye_test/runs/{baseline_preset2,easy_full}`. -- Status: M1 done (pre-flight, staging, child-process odometry with progress and cancel). M2 done (export, below). Next: the M3 validation script (`apps/easy_gui/scripts/check_export.py` already covers fields, range, alignment and the step 2 reference; the ramp-time and near-point-density items are still open). - -### Export (M2, `src/export.cpp`) -- Loads the session with `Session` in `load_cache_mode` (poses from `session_poses.mrp` + trajectories, no points: exactly what step 2 uses), then streams one `scan_lio_N.laz` at a time and then one raw LAZ at a time (near points). The near/output-filter thresholds come from the session's own `HDMapping_params_*.toml` (what step 1 ran with). Peak memory: **102 MB** on the full recording; ~1.5 min. -- Odometry points replicate step 2's `save_all_to_las`: `m_pose ยท (p โˆ’ offset) + offset`, gps_time 0 skipped, scale 0.0001 โ†’ XYZ integer-identical to `multi_view_tls_registration_step_2 --export` (89,326,756 points, max |dXYZ| 0, intensity and point_source_id identical). Trajectory CSV matches step 2's `trajectories.csv` (0.05 mm, 5e-10 quaternion). Step 2's own export writes gps_time ร—1e18 (the session's ns ร—1e9 again). -- `range_m` of odometry points: `|calibโปยน ยท csv_pose[node]โปยน ยท p_local|`, node by step 1's rule on the ramp time; verified against the raw range on coincident pairs (p99.9 0.07 mm). 148 odometry points (0.0002 %) have range 70โ€“99.7 m: steep returns (elevation up to 54ยฐ), since step 1's 70 m limit is horizontal. -- Full recording (hybrid default): 89,326,756 step 1 + 4,038,530 near + 4,026,788 gap points = 97.39 M; 17.1 M of 115.9 M raw points are no-return; point_cloud.laz 948 MB, trajectory.csv 20 MB (114,940 poses); 143 s, peak 106 MB. -- Gap points, validated like the near points (`check_export.py`, per scan against step 1's points of scans sโˆ’1..s+1): NN distance median 19.4 mm, p99 187 mm, vs a control of step 1's own points from the 5 ms before each gap queried without them: median 19.8 mm, p99 192 mm; the same in every range bin (1โ€“3 m 10.6 vs 10.9 mm โ€ฆ 20+ m 68.1 vs 68.2 mm). So the gap points sit where a missing slice of step 1's own points would. 277 gap points are within 0.5 ยตs of a pose time and need a 1 ยตs tolerance when classified from trajectory.csv (1 ยตs resolution). -- Alignment (test mode `--check-band 1.2`, raw points 1.0โ€“1.2 m, now `point_origin = 3`): NN distance to step 1's points median 0.10 mm; with a step 1 counterpart (95.6 %) p90 6.4 mm, p99 12.9 mm; 71 % land on the identical pose. Rebuilding step 1's ramp from the raw files for scans 40โ€“49 reproduces all 7,595,561 step 1 points (โ‰ค 0.17 mm), and shows that the whole difference is step 1's ramp time choosing another node (|ramp โˆ’ raw| p50 1.2 ms, p99 11.6 ms, max 17 ms โ†’ displacement p90 9 mm, max 56 mm). The other 4.4 % are in step 1's inter-chunk gaps (no counterpart). -- `place_by_raw_time` export option (Settings/ExportRequest, `--place-by-raw-time`, GUI advanced checkbox; default **off**): skips scan_lio and places every raw point inside step 1's distance limits from the raw files with the last node before its real gps_time (within 10 ms), writing raw gps_time; near rule unchanged. Full recording: 93,359,291 points inside step 1's limits vs 89,326,756 (+4,026,808 from step 1's inter-chunk gaps, ~5.7 k first-points-of-chunk). Same raw point, default vs raw time (correspondence via rebuilt ramp, both placements verified against the files โ‰ค 0.17 mm): same pose for 79.3 %; median 0 in every bin; p95 12 / 20 / 34 / 69 mm for 1โ€“3 / 3โ€“5 / 5โ€“10 / 10โ€“20 m (moved points only: median 8 / 13 / 23 / 47 mm, p95 19 / 32 / 57 / 118 mm), i.e. a neighbouring node's rotation (~9 mrad). Sharpness (2026-10-17, the same 89,326,756 matched points, selections made on the mean of both positions): - - Ground, 1 ร— 1 m cells within 20 m of the walk, lowest points ยฑ0.15 m band, per-version orthogonal plane fit (5,376 cells): residual std median 65.73 mm default vs 66.14 mm raw time; paired difference (raw โˆ’ default) +0.06 / +0.08 / +0.15 / +0.61 mm at 0โ€“3 / 3โ€“5 / 5โ€“10 / 10โ€“20 m from the path; raw time thinner in only 37 % of cells (Wilcoxon p โ‰ˆ 1e-112). A ยฑ0.05 m band gives the same sign (41 % thinner, +0.05 โ€ฆ +0.24 mm). The ~66 mm is mostly forest-floor roughness, which the paired design cancels. - - Stems, 17 auto-detected and visually checked trunks (1.2โ€“1.4 m above local ground, circle fit to the same points in each version): RMSE median 19.25 vs 19.31 mm, paired +0.05 / +0.43 / +0.03 / +0.17 mm at 1โ€“3 / 3โ€“5 / 5โ€“10 / 10โ€“20 m; raw time sharper for 3 of 17 (p = 0.06). Diameters do not differ systematically: median |ฮ”D| 0.4 mm (< 2 mm within 10 m; โˆ’3 โ€ฆ +7 mm for one-sided arcs at 10โ€“20 m). - - Verdict: default (step 1 placement) slightly sharper, by 0.1โ€“0.6 mm on 20โ€“60 mm thick surfaces; most likely because step 1 optimised the trajectory with its ramp-time points. Scripts: `ground.py`, `stems.py` (not in the repo). -- CloudCompare (2.x CLI) loaded `range_m` (f32) and `near_added` (u8, renamed `point_origin` since) as named scalar fields; laspy reads them as float32/uint8 (laszip_add_attribute would set the scale bit and make them float64, hence the hand-written VLR). - -### Test data -`~/data/mandeye_test/continousScanning_0012`: a local copy of `/Volumes/mandeye/continousScanning_0012` without `CAMERA_0`. 116 `lidar*.laz`, 115 `imu*.csv`/`*.sn`/`status*.json`, single lidar, no calibration file. **Never write to `/Volumes/mandeye`.** - -### `load_data` file handling -Sorts all files, then pairs `lidar*.laz` with `imu*.csv` **by position** and loads `min(#laz, #csv)` pairs; a 4-digit index mismatch only prints a warning, and surplus trailing files are skipped. `status*`/`cam*` JSON is ignored; any other `.json` or a `.mjc` is the calibration file; only the first `.sn` is used. Writes `cache/` and `preview/` into the folder holding the input files. - -### Validation items (dev script) -- Easy GUI export vs `multi_view_tls_registration_step_2 --export`: point count, XYZ, intensity, gps_time, trajectory. -- Ramp-time error: for each odometry chunk, rebuild step 1's ramp from the raw points and compare it with their real gps_time. Report it twice: as step 1 does it (no-return (0,0,0) points removed by the load filter *before* the ramp) and with those points kept, to quantify how much the removal contributes. -- Near points: counts added, range histogram, sensor-frame density below 1.5 m. - -### What step 1 does to points (verified in code, `compute_step_1` / `save_result`) -- **Timestamps are not preserved.** In each 20-pose (โ‰ˆ0.1 s) worker chunk, every point's timestamp is overwritten with a uniform ramp `t0 + kยท(t_last โˆ’ t0)/N` (`lidar_odometry.cpp`, "correct points timestamps"). Raw gps_time cannot be joined to output points, with or without a tolerance. -- Output gps_time in `scan_lio_N.laz` and timestamps in `trajectory_lio_N.csv` are seconds ร— 1e9 (ns stored as a double, โ‰ˆ256 ns resolution). -- `scan_lio_N.laz` carries only XYZ (frame of the scan's first trajectory node), intensity and gps_time. Lidar id is dropped in `points_to_vector`. -- Voxel decimation (`params.decimation`) affects only the points used for optimization; `scan_lio_N.laz` is written from the undecimated originals. Points are still dropped by: the load filter (gps_time == 0; horizontal radius outside [filter_threshold_xy_inner, _outer]); points before the first `threshold_initial_points`; the gap `[t_last, t_next_first)` between consecutive worker chunks (โ‰ˆ1 of 20 IMU intervals, ~5 %); the first point of each chunk; chunks with โ‰ค 1000 points; the trailing poses that don't fill a whole chunk; points with sensor-frame norm โ‰ค `threshould_output_filter`; and the last concatenated group if it has โ‰ค `min_counter_concatenated_trajectory_nodes` nodes. -- Distance filters: `filter_threshold_xy_inner/_outer` act at load (`load_point_cloud`), so they remove points from odometry input *and* output; they measure the **horizontal** radius โˆš(xยฒ+yยฒ) in the raw sensor frame. `threshould_output_filter` is **output-only** (`points_to_vector` in `save_result`, `fill_session_from_worker_data`; the optimizers never read it) and measures the **3D** norm in the sensor frame. Consequence: an output filter below `filter_threshold_xy_inner` has no effect, because every surviving point already has 3D norm โ‰ฅ its horizontal radius > inner. -- Per-point range is recoverable without the raw files: `index_pose` = last trajectory node with t_node < t_point (step 1's own rule, applied to the ramp time), then `p_sensor = csv_pose[index_pose]โปยน ยท p_laz` (both are relative to the scan's first node). -- `save_index_pose` is not in TomlIO's attribute list, so it cannot be enabled through a parameter file. - -### Headless step 1 -`lidar_odometry_step_1 ` runs odometry without a window and calls `save_results` + `remove_cache`. Quirks: the exit code is 0 after a caught exception, so check for `session.mjs` instead; `load_data` waits on stdin before `std::exit` when the calibration doesn't match (give the child a closed stdin); `bad_alloc` opens a native message box; progress appears only as log lines (`running iterations: i/n` during setup, `finished optimizing โ€ฆ - i / N โ€ฆ` per odometry chunk). \ No newline at end of file diff --git a/apps/easy_gui/docs/DESIGN.md b/apps/easy_gui/docs/DESIGN.md new file mode 100644 index 00000000..05e0a764 --- /dev/null +++ b/apps/easy_gui/docs/DESIGN.md @@ -0,0 +1,399 @@ +# Easy GUI: design notes + +This document explains how Easy GUI works, why it is built the way it is, and how its output was validated. It is for +collaborators and reviewers who know HDMapping. The user guide is [../README.md](../README.md). + +Contents: + +1. [Goal and scope](#1-goal-and-scope) +2. [Architecture](#2-architecture) +3. [Background: Mandeye raw data](#3-background-mandeye-raw-data) +4. [Background: what lidar odometry (step 1) does to points](#4-background-what-lidar-odometry-step-1-does-to-points) +5. [Design decisions](#5-design-decisions) +6. [The export](#6-the-export) +7. [Validation](#7-validation) +8. [Experimental option: place every point by its raw time](#8-experimental-option-place-every-point-by-its-raw-time) +9. [Resources: time, memory, disk](#9-resources-time-memory-disk) +10. [Status and next steps](#10-status-and-next-steps) + +--- + +## 1. Goal and scope + +One window that takes a raw Mandeye recording folder, runs HDMapping's lidar odometry (step 1) with the precision +forestry preset, and exports two files: + +- a LAZ point cloud with every per-point attribute that is available, and +- a trajectory CSV with timestamp, position, orientation and every other available field. + +Constraints: + +- Built on the raylib + rlImGui GUI stack (like steps 2 and 3). +- Reuse HDMapping's code; do not duplicate its algorithms. `core_math` is not modified, and the existing apps behave + as before. +- Show progress and report errors in plain language. + +Not in scope (v1): step 2 refinement, step 3 (multi-session), georeferencing (GNSS, control points), and rigs with +more than one lidar. The pre-flight check refuses recordings with more than one lidar id. + +Planned for v2: pose-graph loop closure behind a toggle, using `PoseGraphLoopClosure` with automatically detected +candidates, where an edge is rejected if its ICP fails. `PoseGraphSLAM::optimize` will not be used. + +## 2. Architecture + +``` +recording folder (read only) + โ”‚ + โ”œโ”€ pre-flight check โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ file pairing, LAZ/IMU readability, lidar ids, calibration, clocks, disk space + โ”‚ + โ”œโ”€ staging โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ output/work/staging/: symlinks to the recording's files + โ”‚ + โ”œโ”€ lidar_odometry_step_1 โ”€โ”€โ”€โ”€โ”€ child process: + โ”‚ (forestry preset) progress parsed from its log + โ”‚ + โ””โ”€ export โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ point_cloud.laz, trajectory.csv, README.txt +``` + +Code in `apps/easy_gui`: + +| Part | Role | +|---|---| +| `easy_gui_core` (static library, no raylib) | Everything except the window, so the doctest suite can link it. | +| `src/preflight.*` | Classifies files exactly like step 1's `load_data` does and checks them (see ยง3). | +| `src/staging.*` | Builds the staging folder of symlinks (see ยง5.6). | +| `src/child_process.*` | Runs step 1: POSIX `posix_spawn` with its own process group; Windows `CreateProcess` with a job object (written, not yet compiled on Windows). | +| `src/progress_parser.*` | Turns step 1's log lines into phases and a progress fraction. | +| `src/forestry_preset.*` | Writes the step 1 parameter file with step 1's own `TomlIO`. | +| `src/export.*`, `src/las14_writer.*` | The export (ยง6) and a streaming LAS 1.4 point format 6 LAZ writer with a hand-written extra-bytes VLR. | +| `src/pipeline.*` | Worker thread that runs the stages above and publishes status for the GUI. | +| `src/easy_gui_main.cpp` | The window, plus the command-line modes below. | +| `tests/` | `easy_gui_tests` (doctest). | +| `scripts/check_export.py` | Acceptance checks for an export (ยง7). | + +`easy_gui_core` compiles two step 1 sources by path, like the other apps do: `lidar_odometry_utils.cpp` (for the +calibration parser the pre-flight check uses) and `toml_io.cpp` (to write the parameter file and to read it back from +the session). It is built with `WITH_GUI=0` and links `core_no_gui` (for `Session`) and `core_math`, not `core`. +Because of that, `core_no_gui` is now always built; before, it was only built with tests, pybind or the CLI tools. + +**Why step 1 runs as a child process.** Crash isolation (step 1 calls `std::exit` on some input errors and opens a +native message box on `bad_alloc`), and a real cancel: killing the process group stops it immediately and frees its +~8.5 GB. Building `easy_gui` also builds `lidar_odometry_step_1` (`add_dependencies`). Easy GUI finds it next to its own +executable, under the plain name or the `HDMAPPING_BINARY_PREFIX` name; `HDMAPPING_STEP1_EXECUTABLE` overrides the +lookup. + +Quirks of headless step 1 (`lidar_odometry_step_1 `) that Easy GUI handles: + +- The exit code is 0 even after a caught exception, so success is judged by the presence of `session.mjs`. +- `load_data` waits on stdin before `std::exit` when the calibration does not match, so the child gets a closed stdin. + The pre-flight check repeats `load_data`'s calibration checks so that this case is refused before step 1 starts. +- Progress exists only as log lines: `running iterations: i/n` during setup and `finished optimizing โ€ฆ - i / N โ€ฆ` per + odometry chunk. + +**Command-line modes** of `easy_gui`: + +| Mode | Purpose | +|---|---| +| `--headless ` | The whole pipeline without a window. Exit code 0 success, 1 failed or refused, 2 usage. | +| `--export-only ` | Only the export, on an existing step 1 session (about 2.5 min instead of about 22). | +| `--check-band ` (with either of the above) | Test mode for the alignment check, see ยง7.3. | +| `--place-by-raw-time` (with either of the above) | The experimental export option of ยง8. | +| `--screenshot [recording]` | Renders the window for about 2 s, saves it and exits (for documentation). raylib saves into the working directory. | + +**Output folder:** + +| Path | Content | +|---|---| +| `point_cloud.laz` | The point cloud (ยง6). | +| `trajectory.csv` | The trajectory (ยง6.4). | +| `README.txt` | Field descriptions, time base and near-point rule, generated with the run's own values. | +| `easy_gui.log` | Everything, including step 1's full output. | +| `work/easy_gui_step1.toml` | The parameters step 1 ran with. | +| `work/lio_session/` | The step 1 session (`session.mjs` etc.). Kept, so it can be opened in step 2 for refinement or georeferencing. | +| `work/staging/` | Removed after success unless "Keep intermediate files" is ticked. | + +After a failure or a cancel, `work/` is removed, except that an export failure keeps `work/lio_session/`, since +odometry took most of the run. + +## 3. Background: Mandeye raw data + +Verified on a real recording and in the Mandeye firmware source. + +**LAZ files** (`lidar*.laz`): LAS 1.2, point format 1. + +- Real fields: X, Y, Z in the sensor frame, intensity (0โ€“160 in practice), gps_time, and user_data = lidar id. There + are no extra bytes; every other standard field is 0. +- gps_time is Unix epoch seconds. The header wrongly flags it as adjusted GPS time. +- classification is a constant 20 and has no meaning: the firmware never copies the Livox point tag. A planned firmware + fix will store the tag there (bits 0โ€“1 adhesion, 2โ€“3 particles, 4โ€“5 other). +- About 15 % of the records are "no return" points: exactly (0, 0, 0) or gps_time 0. + +**IMU files** (`imu*.csv`, 200 Hz): timestamp (ns, the same clock as the LAZ gps_time), gyro XYZ, acc XYZ (g), imuId, +and timestampUnix (ms, the host computer's clock, not synchronised with the others). + +**How step 1's `load_data` picks files** (the pre-flight check mirrors this): + +- It sorts all files, then pairs `lidar*.laz` with `imu*.csv` **by position**, and loads `min(#laz, #csv)` pairs. A + mismatch of the 4-digit index only prints a warning; surplus trailing files are skipped. +- `status*` and `cam*` JSON files are ignored. Any other `.json`, or a `.mjc`, is taken as the calibration file. Only + the first `.sn` file is used. +- It writes `cache/` and `preview/` into the folder that holds the input files. + +## 4. Background: what lidar odometry (step 1) does to points + +Verified in `compute_step_1` and `save_result` (`apps/lidar_odometry_step_1`). These facts drive most decisions in ยง5. + +**Timestamps are not preserved.** Step 1 works in worker chunks of 20 trajectory poses (โ‰ˆ 0.1 s). In each chunk it +overwrites every point's timestamp with a uniform ramp `t0 + kยท(t_last โˆ’ t0)/N` ("correct points timestamps" in +`lidar_odometry.cpp`). Raw gps_time therefore cannot be joined to step 1's output points, with or without a tolerance. + +Output formats: gps_time in `scan_lio_N.laz` and the timestamps in `trajectory_lio_N.csv` are seconds ร— 1e9 (ns stored +as a double, โ‰ˆ 256 ns resolution). `scan_lio_N.laz` holds only XYZ (in the frame of the scan's first trajectory node), +intensity and gps_time; the lidar id is dropped in `points_to_vector`. + +**Distance filters:** + +- `filter_threshold_xy_inner` / `_outer` act at load time (`load_point_cloud`), so they remove points from the odometry + input *and* from the output. They measure the **horizontal** radius โˆš(xยฒ + yยฒ) in the raw sensor frame. +- `threshould_output_filter` is **output only** (`points_to_vector` in `save_result`, `fill_session_from_worker_data`; + the optimisers never read it) and measures the **3D** norm in the sensor frame. +- Consequently an output filter below `filter_threshold_xy_inner` has no effect: every surviving point already has a 3D + norm โ‰ฅ its horizontal radius > inner. + +**Other points step 1 drops:** gps_time 0; points before the first `threshold_initial_points`; the gap +`[t_last, t_next_first)` between consecutive worker chunks (โ‰ˆ 1 of every 20 IMU intervals, about 5 % of the time); the +first point of each chunk; chunks with โ‰ค 1000 points; the trailing poses that do not fill a whole chunk; and the last +concatenated group if it has โ‰ค `min_counter_concatenated_trajectory_nodes` nodes. + +Voxel decimation (`params.decimation`) only thins the points used for optimisation; `scan_lio_N.laz` is written from +the undecimated originals. + +**Per-point range is recoverable without the raw files:** `index_pose` = the last trajectory node with t_node < t_point +(step 1's own rule, applied to the ramp time), then `p_sensor = csv_pose[index_pose]โปยน ยท p_laz`, both relative to the +scan's first node. (`save_index_pose` would record the index directly, but it is not in `TomlIO`'s attribute list, so a +parameter file cannot enable it.) + +## 5. Design decisions + +### 5.1 Output format: LAS 1.4, point format 6, extra bytes + +Point format 6 has a 64-bit gps_time and room for extra bytes. Two extra-bytes attributes are written: + +- `range_m` (float32): 3D distance from the lidar at the moment of measurement. +- `point_origin` (uint8): where the point comes from (0 = step 1, 1 = near point, 2 = gap point, 3 = test mode only). + +The extra-bytes VLR is written by hand because `laszip_add_attribute` sets the scale bit, which makes readers such as +laspy decode the fields as float64. With the hand-written VLR, CloudCompare 2.x loads both as named scalar fields and +laspy reads them as float32 / uint8. + +`user_data` keeps the lidar id; `point_source_id` is the step 1 scan index (the `scan_lio_N.laz` number, about 5 s of +recording). + +### 5.2 Time base + +`gps_time` is Unix epoch seconds, not GPS week time and not adjusted standard GPS time, so global encoding bit 0 (GPS +time type) is 0. README.txt states this. For the default export, step 1's own points carry step 1's ramp time (ยง4), +typically 1โ€“4 ms and at most about 20 ms from the real measurement time; near and gap points carry their raw time. + +(Step 2's own `--export` writes gps_time ร— 1e18, because it multiplies the session's ns-valued times by 1e9 again. +Easy GUI does not reproduce that.) + +### 5.3 The default export is "hybrid" + +The point cloud combines: + +- `point_origin = 0`: step 1's own points, placed exactly as step 1 placed them; +- `point_origin = 1`: **near points**, raw points step 1's distance filters removed (ยง5.4); +- `point_origin = 2`: **gap points**, raw points inside step 1's distance limits that step 1 skipped between its 0.1 s + pieces (ยง5.5). + +Near and gap points come from the raw files and are placed at their real gps_time with step 1's rule: the last +trajectory node with t_node < t, if it is within 10 ms (step 1's own tolerance), so +`global = mrp[scan] ยท csv_pose[node] ยท calib ยท p_raw`. + +Placing step 1's own points by raw time as well is possible (ยง8) but measured slightly less sharp, so it is only an +experimental option. + +### 5.4 Near points + +Step 1 drops every point whose horizontal range is below 1.0 m, which leaves a hole around the walking path. Easy GUI +adds those points back at export; odometry never sees them. + +Rule: keep raw points with `near_min_range` โ‰ค 3D range **and** that step 1's distance filters removed. As implemented +this is the exact complement of step 1's filters: horizontal range โ‰ค `filter_threshold_xy_inner` (1.0 m) or 3D norm โ‰ค +`threshould_output_filter`, and horizontal range < `filter_threshold_xy_outer`. No-return points are always dropped. + +`near_min_range` defaults to 0.4 m (advanced setting "Closest points kept", 0.1โ€“0.95 m). In measured raw data the +operator's body and the handle show up only as small blobs inside 0.4 m, and near points beyond that are environment in +every direction. + +**Output filter.** For the near rule to meet step 1's output without a gap, Easy GUI sets +`threshould_output_filter = filter_threshold_xy_inner` (1.0 m instead of the preset's 1.5 m). Because the output filter +is output only (ยง4), the trajectory is unchanged; this was confirmed byte for byte (ยง7.1). Points with horizontal range +โ‰ฅ 1.0 m then all come from step 1, and points below come from the near rule. + +### 5.5 Gap points + +Step 1 takes `[first node, last node)` of each 0.1 s piece, so the raw points between a piece's last node and the next +piece's first node are never exported (about 5 % of the recording time). Easy GUI adds them back: a raw point inside +step 1's distance limits is a gap point when the last trajectory node at or before its time is the last node of a +piece, i.e. `j % threshold_nr_poses == threshold_nr_poses โˆ’ 1` for node j within the scan. + +### 5.6 Never write into the recording + +The recording may be on the scanner's USB stick, so Easy GUI never writes into it. Because `load_data` writes `cache/` +and `preview/` next to its input files (ยง3), step 1 runs on a staging folder of symlinks inside the Easy GUI output +folder. The pre-flight check refuses an output folder inside the recording. + +### 5.7 Forestry preset + +The preset is `LidarOdometryParams` defaults plus the 18 overrides of "Presets โ†’ 2 โ€ฆ precise forestry" in +`lidar_odometry_gui.cpp`. HDMapping has no shared preset function (the values are inline GUI code, also copied into the +`precision_forestry` app), so `forestry_preset.cpp` repeats them and writes the TOML through step 1's own `TomlIO`. +The only deviation is the output filter of ยง5.4. + +## 6. The export + +Source: `src/export.cpp`. + +### 6.1 Flow and memory + +1. Load the session with `Session` in `load_cache_mode`: poses from `session_poses.mrp` plus the trajectories, no + points. This is exactly what step 2 uses. +2. Stream one `scan_lio_N.laz` at a time (step 1's points). +3. Stream one raw LAZ at a time (near and gap points). + +The distance thresholds are read from the session's own `HDMapping_params_*.toml`, i.e. what step 1 actually ran with. +Peak memory on the full test recording: 106 MB. + +### 6.2 Step 1's points + +These replicate step 2's `save_all_to_las`: `m_pose ยท (p โˆ’ offset) + offset`, gps_time 0 skipped, scale 0.0001. The +result is integer-identical to `multi_view_tls_registration_step_2 --export` (ยง7.2). + +`range_m` of a step 1 point is `|calibโปยน ยท csv_pose[node]โปยน ยท p_local|`, with the node chosen by step 1's rule on the +ramp time. + +### 6.3 Point counts on the test recording + +10-minute forest walk, 115 file pairs, 115.9 M raw points of which 17.1 M are no-return records. + +| point_origin | Points | +|---|---| +| 0, step 1 | 89,326,756 | +| 1, near | 4,038,530 | +| 2, gap | 4,026,788 | +| total | 97.39 M | + +point_cloud.laz 948 MB, trajectory.csv 20 MB (114,940 poses). Export time 143 s. + +148 step 1 points (0.0002 %) have a range of 70โ€“99.7 m. These are steep returns (elevation up to 54ยฐ): step 1's 70 m +limit is horizontal. + +### 6.4 Trajectory CSV + +One row per trajectory node: `timestamp_unix_s`, `x, y, z`, `qw, qx, qy, qz`, `om_deg, fi_deg, ka_deg` (HDMapping +Tait-Bryan angles), `scan_index` (= `point_source_id`), `host_clock_unix_s` (the IMU file's host clock, ms resolution, +not synchronised), and `imu_om_deg, imu_fi_deg, imu_ka_deg` (IMU-only orientation before odometry). Timestamps have +1 ยตs resolution. + +## 7. Validation + +`scripts/check_export.py [--step2 ]` automates the checks below (fields, range, alignment, gap +points, step 2 reference). It needs `laspy[lazrs]`, numpy and scipy. All numbers are from the test recording of ยง6.3. + +### 7.1 Easy GUI's step 1 run is the manual one + +Easy GUI (output filter 1.0 m) against a manual step 1 run with preset 2 (1.5 m): the trajectory is **byte-identical** +(`session_poses.mrp`, `trajectory_lio_*.csv`). So step 1 is deterministic here, and the output filter is confirmed to be +output only. `scan_lio` has 89.33 M points instead of 84.21 M (+5.11 M, +6.1 %), the points between 1.0 and 1.5 m. + +### 7.2 Step 1's points match step 2's export + +Against `multi_view_tls_registration_step_2 --export`: 89,326,756 points in both, max |ฮ”XYZ| = 0 (integer +coordinates identical), intensity and point_source_id identical. The trajectory CSV matches step 2's `trajectories.csv` +to 0.05 mm and 5e-10 in the quaternion. + +`range_m` of step 1's points, compared with the raw range on coincident pairs: p99.9 difference 0.07 mm. + +### 7.3 Near points line up with step 1's points + +Test mode `--check-band 1.2` also exports the raw points with horizontal range 1.0โ€“1.2 m, which step 1 exported too, +placed the near-point way and flagged `point_origin = 3`. Their nearest-neighbour distance to step 1's points: + +- median 0.10 mm overall; +- for the 95.6 % that have a step 1 counterpart: p90 6.4 mm, p99 12.9 mm; 71 % land on the identical pose; +- the other 4.4 % lie in step 1's inter-chunk gaps, so they have no counterpart. + +Rebuilding step 1's ramp from the raw files for scans 40โ€“49 reproduces all 7,595,561 step 1 points (โ‰ค 0.17 mm) and +shows that the whole difference comes from step 1's ramp time choosing a different node than the real time would: +|ramp โˆ’ raw| p50 1.2 ms, p99 11.6 ms, max 17 ms, giving displacements of p90 9 mm, max 56 mm. + +### 7.4 Gap points sit where a missing slice of step 1's points would + +Per scan, against step 1's points of scans sโˆ’1 โ€ฆ s+1, gap points have a nearest-neighbour distance of median 19.4 mm, +p99 187 mm. Control: step 1's own points from the 5 ms before each gap, queried without them: median 19.8 mm, p99 +192 mm. The same holds in every range bin (1โ€“3 m: 10.6 vs 10.9 mm โ€ฆ 20+ m: 68.1 vs 68.2 mm). + +277 gap points are within 0.5 ยตs of a pose time; classifying points from trajectory.csv (1 ยตs resolution) needs a +1 ยตs tolerance for them. + +## 8. Experimental option: place every point by its raw time + +`place_by_raw_time` (GUI advanced checkbox, `--place-by-raw-time`; default **off**) skips `scan_lio` and places every +raw point inside step 1's distance limits from the raw files, with the last node before its real gps_time (within +10 ms), and writes the raw gps_time. The near rule is unchanged. + +On the test recording this gives 93,359,291 points inside step 1's limits instead of 89,326,756 (+4,026,808 from step +1's inter-chunk gaps, plus ~5.7 k first points of chunks). + +**Displacement against the default.** Same raw point in both versions (matched via the rebuilt ramp; both placements +verified against the files to โ‰ค 0.17 mm): the same pose for 79.3 % of points, median displacement 0 in every range bin. +p95 is 12 / 20 / 34 / 69 mm at 1โ€“3 / 3โ€“5 / 5โ€“10 / 10โ€“20 m (moved points only: median 8 / 13 / 23 / 47 mm, p95 +19 / 32 / 57 / 118 mm), which is a neighbouring node's rotation (~9 mrad). + +**Sharpness** (2026-10-17; the same 89,326,756 matched points in both versions, selections made on the mean of both +positions, so the comparison is paired): + +| Test | Default | Raw time | Paired difference (raw โˆ’ default) by distance from the path | Raw time better in | +|---|---|---|---|---| +| Ground: 5,376 cells of 1 ร— 1 m within 20 m of the walk, lowest points in a ยฑ0.15 m band, orthogonal plane fit, residual std | median 65.73 mm | 66.14 mm | +0.06 / +0.08 / +0.15 / +0.61 mm at 0โ€“3 / 3โ€“5 / 5โ€“10 / 10โ€“20 m | 37 % of cells (Wilcoxon p โ‰ˆ 1e-112) | +| Ground, ยฑ0.05 m band | | | +0.05 โ€ฆ +0.24 mm | 41 % | +| Stems: 17 automatically detected, visually checked trunks, circle fit at 1.2โ€“1.4 m above local ground, RMSE | median 19.25 mm | 19.31 mm | +0.05 / +0.43 / +0.03 / +0.17 mm at 1โ€“3 / 3โ€“5 / 5โ€“10 / 10โ€“20 m | 3 of 17 (p = 0.06) | + +The ~66 mm ground thickness is mostly forest-floor roughness, which the paired design cancels. Stem diameters do not +differ systematically: median |ฮ”D| 0.4 mm (< 2 mm within 10 m; โˆ’3 โ€ฆ +7 mm for one-sided arcs at 10โ€“20 m). + +**Verdict:** the default (step 1's placement) is slightly sharper, by 0.1โ€“0.6 mm on 20โ€“60 mm thick surfaces. The most +likely reason is that step 1 optimised the trajectory with its own ramp-time points. Hence the option stays off. (The +ground and stem analysis scripts are not in the repository.) + +## 9. Resources: time, memory, disk + +Measured on the test recording (117 M raw points, 10-minute walk): + +- Step 1 takes ~21 min with a peak memory footprint of 8.5 GB; the export ~2.5 min (143 s) with ~106 MB. +- Step 1 needs ~108 B per raw point of temporary disk at its peak (cache 7.5 GB + chunk caches ~4 GB + `scan_lio` for + 117 M points). +- Results are 2.14ร— the raw LAZ size: session 0.82ร— + point_cloud.laz 1.29ร— (with near and gap points) + + trajectory.csv 0.03ร—. + +The pre-flight check therefore estimates 120 B per raw point of temporary space plus 2.25ร— the raw LAZ size for results, +always reports the breakdown, warns below 1.5ร— the estimate and refuses below 1ร—. + +## 10. Status and next steps + +- **M1 (done):** pre-flight check, staging, step 1 as a child process with progress and cancel. +- **M2 (done):** export (ยง6). +- **M3 (in progress):** validation script. `check_export.py` covers fields, range, alignment, gap points and the step 2 + reference. Still open: + - Ramp-time error: for each odometry chunk, rebuild step 1's ramp from the raw points and compare with their real + gps_time, reported twice: as step 1 does it (no-return points removed by the load filter *before* the ramp) and + with those points kept, to quantify how much their removal contributes. + - Near points: counts, range histogram, and sensor-frame density below 1.5 m. +- **Windows:** `child_process.cpp` has a Windows implementation (CreateProcess + job object) that has not been compiled + yet. +- **v2:** gated pose-graph loop closure (ยง1). + +Tests: `easy_gui_tests` (doctest, synthetic recordings and sessions). The integration tests against the real step 1 +binary run only when `EASY_GUI_TEST_RECORDING=` is set; a folder of symlinks to the first 4 file +pairs of a recording is enough. diff --git a/apps/easy_gui/src/easy_gui_main.cpp b/apps/easy_gui/src/easy_gui_main.cpp index 9b97c7d7..8dbc52fe 100644 --- a/apps/easy_gui/src/easy_gui_main.cpp +++ b/apps/easy_gui/src/easy_gui_main.cpp @@ -1,5 +1,5 @@ // HDMapping Easy GUI: raw Mandeye recording -> lidar odometry (precision forestry preset) -> point cloud + trajectory. -// See CLAUDE.md ("Easy GUI") for the decisions behind the pipeline. +// See apps/easy_gui/docs/DESIGN.md for the decisions behind the pipeline. #include "export.h" #include "pipeline.h" diff --git a/apps/easy_gui/src/export.h b/apps/easy_gui/src/export.h index d72854be..ea7942fe 100644 --- a/apps/easy_gui/src/export.h +++ b/apps/easy_gui/src/export.h @@ -33,7 +33,7 @@ namespace easy_gui //! taking step 1's points (scan_lio_N.laz, placed at step 1's ramp time). Same distance filters, near-point and //! gap-point rules; raw gps_time is written. point_origin = 0 then means "inside step 1's distance limits, and //! not in a gap". check_band_outer_m is ignored (those points are all included). Measured slightly less sharp - //! than the default (see CLAUDE.md), hence off. + //! than the default (see docs/DESIGN.md), hence off. bool place_by_raw_time = false; std::filesystem::path laz_file; std::filesystem::path trajectory_file; diff --git a/apps/easy_gui/src/forestry_preset.h b/apps/easy_gui/src/forestry_preset.h index 1e107c74..7cb110ff 100644 --- a/apps/easy_gui/src/forestry_preset.h +++ b/apps/easy_gui/src/forestry_preset.h @@ -17,7 +17,7 @@ namespace easy_gui //! Easy GUI's only deviation from the preset: threshould_output_filter (output-only) is lowered to //! filter_threshold_xy_inner, so step 1 exports every point the load filter kept, and the near-point export picks - //! up exactly the points the load filter removed. The trajectory is unaffected (see CLAUDE.md). + //! up exactly the points the load filter removed. The trajectory is unaffected (see docs/DESIGN.md). void applyEasyGuiOutputSettings(LidarOdometryParams& params); //! Writes the parameter file the step 1 child process runs with (value-initialised params + preset + Easy GUI From da696da7e4b4f0bdccd1042de0853f37050aadb2 Mon Sep 17 00:00:00 2001 From: Martin Mokros Date: Sun, 18 Oct 2026 11:33:16 +0100 Subject: [PATCH 06/23] Add HDMAPPING_EASY_GUI_ONLY build switch (default off) With -DHDMAPPING_EASY_GUI_ONLY=ON only easy_gui, lidar_odometry_step_1 and the libraries they link are built (plus easy_gui_tests with BUILD_TESTING), and only those two programs are installed. The library directories and the step 1 directory are added EXCLUDE_FROM_ALL; all other apps, test suites and the optional tool sets are skipped. With the default (off) the build is unchanged. Co-Authored-By: Claude Opus 5.5 --- CMakeLists.txt | 43 ++++++++++++++++++++++++++++++++++++------- 1 file changed, 36 insertions(+), 7 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index dc133cc3..b66a4628 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -32,6 +32,7 @@ option(BUILD_WITH_BUNDLED_LIBLASZIP "Build with bundled Lib LASZIP" ON) option(BUILD_WITH_BUNDLED_ONETBB "Build with bundled oneTBB" ON) option(BUILD_WITH_PYBIND "Enable pybind11 bindings" OFF) option(HDMAPPING_ASSERT_ALWAYS_ABORT "Make HDMAPPING_ASSERT() abort on failure even in release (NDEBUG) builds" OFF) +option(HDMAPPING_EASY_GUI_ONLY "Build only easy_gui, lidar_odometry_step_1 and their dependencies (plus easy_gui_tests with BUILD_TESTING)" OFF) add_definitions(-D_USE_MATH_DEFINES) @@ -107,18 +108,34 @@ include(cmake/dependencies.cmake) # unconditionally: apps/multi_view_tls_registration (step2) is raylib-based. include(cmake/raylib.cmake) +# HDMAPPING_EASY_GUI_ONLY: the library directories are still added (easy_gui and +# step 1 link some of their targets), but EXCLUDE_FROM_ALL, so a target in them +# is only built when easy_gui or lidar_odometry_step_1 needs it. All other apps, +# tests and optional tool sets are skipped. Off by default: nothing changes. +if(HDMAPPING_EASY_GUI_ONLY) + set(HDMAPPING_EASY_GUI_ONLY_EXCLUDE EXCLUDE_FROM_ALL) + foreach(_hd_skipped BUILD_WITH_CLI_TOOLS BUILD_WITH_HD_MAPPER_APPLICATION BUILD_WITH_UTILITY_APPLICATION) + if(${_hd_skipped}) + message(STATUS "HDMAPPING_EASY_GUI_ONLY: ignoring ${_hd_skipped}") + endif() + endforeach() +endif() + # ============================================================================ # Core Library # ============================================================================ -add_subdirectory(core) -add_subdirectory(calib_core) -add_subdirectory(raylib_widgets) +add_subdirectory(core ${HDMAPPING_EASY_GUI_ONLY_EXCLUDE}) +add_subdirectory(calib_core ${HDMAPPING_EASY_GUI_ONLY_EXCLUDE}) +add_subdirectory(raylib_widgets ${HDMAPPING_EASY_GUI_ONLY_EXCLUDE}) # ============================================================================ # Unit tests (opt-in: cmake -DBUILD_TESTING=ON) # ============================================================================ option(BUILD_TESTING "Build HDMapping unit tests" OFF) -if(BUILD_TESTING) +if(BUILD_TESTING AND HDMAPPING_EASY_GUI_ONLY) + enable_testing() + add_subdirectory(apps/easy_gui/tests) +elseif(BUILD_TESTING) enable_testing() add_subdirectory(shared/tests) add_subdirectory(calib_core/tests) @@ -130,15 +147,26 @@ endif() set(CORE_LIBRARIES core) set(GUI_LIBRARIES imgui imguizmo implot) -if(BUILD_WITH_CLI_TOOLS) +if(HDMAPPING_EASY_GUI_ONLY) + # The step 1 directory also defines the drag-and-drop precision forestry app. + # EXCLUDE_FROM_ALL skips it, together with the directory's install rules, so + # lidar_odometry_step_1 is put back into "all" and installed from here. + add_subdirectory(apps/lidar_odometry_step_1 EXCLUDE_FROM_ALL) + set_target_properties(lidar_odometry_step_1 PROPERTIES EXCLUDE_FROM_ALL OFF) + hdmapping_install_app(lidar_odometry_step_1) + add_subdirectory(apps/easy_gui) +endif() + +if(BUILD_WITH_CLI_TOOLS AND NOT HDMAPPING_EASY_GUI_ONLY) add_subdirectory(apps/console_tools) endif() -if(BUILD_WITH_HD_MAPPER_APPLICATION) +if(BUILD_WITH_HD_MAPPER_APPLICATION AND NOT HDMAPPING_EASY_GUI_ONLY) add_subdirectory(apps/hd_mapper) endif() # Applications for typical workflow - see images/workflow.png +if(NOT HDMAPPING_EASY_GUI_ONLY) add_subdirectory(apps/mandeye_raw_data_viewer) add_subdirectory(apps/lidar_odometry_step_1) add_subdirectory(apps/mandeye_single_session_viewer) @@ -149,8 +177,9 @@ add_subdirectory(apps/multi_session_registration_legacy) add_subdirectory(apps/session_uncertainty_calculation) add_subdirectory(apps/session_to_session_rigid_registration) add_subdirectory(apps/easy_gui) +endif() -if(BUILD_WITH_UTILITY_APPLICATION) +if(BUILD_WITH_UTILITY_APPLICATION AND NOT HDMAPPING_EASY_GUI_ONLY) add_subdirectory(apps/split_multi_livox) add_subdirectory(apps/precision_forestry_tools) add_subdirectory(apps/compare_trajectories) From 4d556e3d797b9d5eda9d49cca235fe4044252aca Mon Sep 17 00:00:00 2001 From: Martin Mokros Date: Sun, 18 Oct 2026 12:12:40 +0100 Subject: [PATCH 07/23] Easy GUI: user-facing name HDMapping Easy Forestry; macOS 13.0 target for Easy-GUI-only builds Window title, heading, messages, the generated README.txt, the LAS system identifier and the default output folder (Documents/HDMapping Easy Forestry) use the new name. Internal names (apps/easy_gui, easy_gui) are unchanged. HDMAPPING_EASY_GUI_ONLY on macOS now defaults CMAKE_OSX_DEPLOYMENT_TARGET to 13.0, so release builds do not require the build machine's macOS version. Co-Authored-By: Claude Opus 5.5 --- CMakeLists.txt | 6 ++++++ apps/easy_gui/src/easy_gui_main.cpp | 10 +++++----- apps/easy_gui/src/export.cpp | 8 ++++---- apps/easy_gui/src/las14_writer.cpp | 2 +- apps/easy_gui/src/pipeline.cpp | 5 +++-- apps/easy_gui/src/preflight.cpp | 9 +++++---- 6 files changed, 24 insertions(+), 16 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b66a4628..277c431a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,5 +1,11 @@ cmake_minimum_required(VERSION 4.0.0) +# Easy Forestry release builds (HDMAPPING_EASY_GUI_ONLY) must run on older macOS versions than the build machine's, +# which is the default minimum. Must be set before project(). Pass -DCMAKE_OSX_DEPLOYMENT_TARGET to override. +if(HDMAPPING_EASY_GUI_ONLY AND CMAKE_HOST_APPLE AND NOT CMAKE_OSX_DEPLOYMENT_TARGET) + set(CMAKE_OSX_DEPLOYMENT_TARGET "13.0" CACHE STRING "Minimum macOS version" FORCE) +endif() + project(hd-mapping VERSION 0.105.0) # Common settings diff --git a/apps/easy_gui/src/easy_gui_main.cpp b/apps/easy_gui/src/easy_gui_main.cpp index 8dbc52fe..ae641f12 100644 --- a/apps/easy_gui/src/easy_gui_main.cpp +++ b/apps/easy_gui/src/easy_gui_main.cpp @@ -44,11 +44,11 @@ namespace return home ? fs::path(home) : fs::current_path(); } - //! ~/Documents/HDMapping Easy/, with _2, _3, ... appended if that already exists. Never next to + //! ~/Documents/HDMapping Easy Forestry/, with _2, _3, ... appended if that already exists. Never next to //! the recording: that may be the scanner's USB stick. fs::path defaultOutputFolder(const fs::path& recording) { - const fs::path base = homeFolder() / "Documents" / "HDMapping Easy"; + const fs::path base = homeFolder() / "Documents" / "HDMapping Easy Forestry"; const std::string name = recording.filename().empty() ? recording.parent_path().filename().string() : recording.filename().string(); std::error_code ec; fs::path candidate = base / name; @@ -572,7 +572,7 @@ int main(int argc, char* argv[]) app.setRecording(argv[first_positional]); SetConfigFlags(FLAG_WINDOW_RESIZABLE | FLAG_MSAA_4X_HINT); - InitWindow(900, 780, ("HDMapping Easy GUI " + std::string(HDMAPPING_CONFIGURE_PROJECT_VERSION)).c_str()); + InitWindow(900, 780, ("HDMapping Easy Forestry " + std::string(HDMAPPING_CONFIGURE_PROJECT_VERSION)).c_str()); raylib_widgets::fitWindowToScreen(); // raylib's default exit key (Esc) would close the window mid-run. SetExitKey(KEY_NULL); @@ -605,9 +605,9 @@ int main(int argc, char* argv[]) ImGui::SetNextWindowPos(viewport->WorkPos); ImGui::SetNextWindowSize(viewport->WorkSize); ImGui::Begin( - "HDMapping Easy GUI", nullptr, ImGuiWindowFlags_NoDecoration | ImGuiWindowFlags_NoMove | ImGuiWindowFlags_NoSavedSettings); + "HDMapping Easy Forestry", nullptr, ImGuiWindowFlags_NoDecoration | ImGuiWindowFlags_NoMove | ImGuiWindowFlags_NoSavedSettings); - ImGui::TextUnformatted("HDMapping Easy GUI"); + ImGui::TextUnformatted("HDMapping Easy Forestry"); ImGui::TextDisabled("Mandeye recording -> lidar odometry (precision forestry settings) -> point cloud and trajectory"); ImGui::Spacing(); diff --git a/apps/easy_gui/src/export.cpp b/apps/easy_gui/src/export.cpp index a530fbcf..983ab72a 100644 --- a/apps/easy_gui/src/export.cpp +++ b/apps/easy_gui/src/export.cpp @@ -210,9 +210,9 @@ namespace easy_gui std::snprintf(near_min, sizeof(near_min), "%.2f m", request.near_min_range_m); std::snprintf(output_filter, sizeof(output_filter), "%.2f m", stats.threshould_output_filter); - out << "HDMapping Easy GUI results\n" + out << "HDMapping Easy Forestry results\n" << "==========================\n\n" - << "Made by HDMapping Easy GUI " << HDMAPPING_CONFIGURE_PROJECT_VERSION << " from the recording\n" + << "Made by HDMapping Easy Forestry " << HDMAPPING_CONFIGURE_PROJECT_VERSION << " from the recording\n" << " " << request.recording.laz_files.front().parent_path().string() << "\n" << "with lidar odometry (precision forestry settings). No loop closure and no georeferencing: coordinates\n" << "are in metres in a local frame that starts at the scanner's first position (z up, from the IMU).\n\n" @@ -248,7 +248,7 @@ namespace easy_gui << " point_origin extra bytes, uint8: where the point comes from (see below)\n" << (request.place_by_raw_time ? " 0 = inside lidar odometry's distance limits, placed by raw time\n" : " 0 = lidar odometry's own point\n") - << " 1 = near point, 2 = gap point (both added by Easy GUI)\n" + << " 1 = near point, 2 = gap point (both added by Easy Forestry)\n" << (stats.check_band_points ? " 3 = TEST MODE alignment-check point (duplicate of an odometry point)\n" : "") << "\n" @@ -257,7 +257,7 @@ namespace easy_gui << "frame) is between " << inner << " and " << outer << " and whose 3D distance is above " << output_filter << ", and it works in\n" << "0.1 s pieces: it skips the raw points measured after the last pose of a piece and before the first pose\n" - << "of the next one (about 5 % of the recording time). Easy GUI adds both kinds of missing points back from\n" + << "of the next one (about 5 % of the recording time). Easy Forestry adds both kinds of missing points back from\n" << "the raw recording, so the cloud has no hole around the walking path and nothing is missing between the\n" << "pieces.\n\n" << "point_origin = 0 (" << formatCount(stats.odometry_points) << " points): " diff --git a/apps/easy_gui/src/las14_writer.cpp b/apps/easy_gui/src/las14_writer.cpp index 7e1c42dd..ab1da4bb 100644 --- a/apps/easy_gui/src/las14_writer.cpp +++ b/apps/easy_gui/src/las14_writer.cpp @@ -89,7 +89,7 @@ namespace easy_gui header->global_encoding = (1 << 4); // bit 0 = 0: not adjusted GPS time; bit 4: WKT (required for PF6) header->version_major = 1; header->version_minor = 4; - copyField(header->system_identifier, sizeof(header->system_identifier), "HDMapping Easy GUI"); + copyField(header->system_identifier, sizeof(header->system_identifier), "HDMapping Easy Forestry"); copyField(header->generating_software, sizeof(header->generating_software), generating_software); { const std::time_t now = std::time(nullptr); diff --git a/apps/easy_gui/src/pipeline.cpp b/apps/easy_gui/src/pipeline.cpp index 9093b9ba..e7e9d2fd 100644 --- a/apps/easy_gui/src/pipeline.cpp +++ b/apps/easy_gui/src/pipeline.cpp @@ -209,7 +209,7 @@ namespace easy_gui } log_file_.open(out.log_file, std::ios::out | std::ios::app); - log("[easy_gui] HDMapping Easy GUI " + std::string(HDMAPPING_CONFIGURE_PROJECT_VERSION) + ", started " + timestamp()); + log("[easy_gui] HDMapping Easy Forestry " + std::string(HDMAPPING_CONFIGURE_PROJECT_VERSION) + ", started " + timestamp()); log("[easy_gui] recording: " + request_.recording.string()); log("[easy_gui] output: " + out.root.string()); log("[easy_gui] step 1 program: " + request_.step1_executable.string()); @@ -249,7 +249,8 @@ namespace easy_gui cleanupWork(false); finish( PipelineState::Failed, - "Easy GUI could not find the lidar odometry program (lidar_odometry_step_1) next to itself. Please reinstall HDMapping.", + "Easy Forestry could not find the lidar odometry program (lidar_odometry_step_1) next to itself. Please reinstall " + "HDMapping Easy Forestry.", "looked for: " + request_.step1_executable.string()); busy_ = false; return; diff --git a/apps/easy_gui/src/preflight.cpp b/apps/easy_gui/src/preflight.cpp index c67b66a7..b2b93d17 100644 --- a/apps/easy_gui/src/preflight.cpp +++ b/apps/easy_gui/src/preflight.cpp @@ -358,7 +358,7 @@ namespace easy_gui ids += (ids.empty() ? "" : ", ") + std::to_string(id); add(Severity::Error, "This recording contains data from more than one lidar (ids " + ids + - "). Easy GUI supports single-lidar recordings only; use the standard HDMapping apps for this one."); + "). Easy Forestry supports single-lidar recordings only; use the standard HDMapping apps for this one."); } else if (lidar_ids.size() == 1) r.lidar_id = *lidar_ids.begin(); @@ -375,7 +375,7 @@ namespace easy_gui else if (calibration.size() > 1) add(Severity::Error, "The calibration file '" + f.calibration_file.filename().string() + - "' describes more than one lidar. Easy GUI supports single-lidar recordings only."); + "' describes more than one lidar. Easy Forestry supports single-lidar recordings only."); else { for (const auto& [id, sn] : id_to_sn) @@ -391,7 +391,7 @@ namespace easy_gui } if (id_to_sn.size() > 1) add(Severity::Error, - "The scanner serial-number file lists more than one lidar. Easy GUI supports single-lidar recordings only."); + "The scanner serial-number file lists more than one lidar. Easy Forestry supports single-lidar recordings only."); // --- IMU vs lidar clocks double imu_first = 0.0, imu_last = 0.0; @@ -426,7 +426,8 @@ namespace easy_gui add(Severity::Error, "Choose an output folder."); else if (isSameOrInside(output_folder, recording)) add(Severity::Error, - "The output folder is inside the recording folder. Choose a folder elsewhere: Easy GUI never writes into the recording " + "The output folder is inside the recording folder. Choose a folder elsewhere: Easy Forestry never writes into the " + "recording " "(it may be on the scanner's USB stick)."); else { From 8b8bc831c2447f23ddcac4e418f34b809e8458c8 Mon Sep 17 00:00:00 2001 From: Martin Mokros Date: Sun, 18 Oct 2026 12:12:40 +0100 Subject: [PATCH 08/23] Easy Forestry: user README and design notes update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit apps/easy_gui/README.md: user guide for non-experts (requirements, installation incl. first-launch warnings, the four steps with a screenshot, output fields, CloudCompare, troubleshooting, getting help, citation). DESIGN.md: naming, macOS deployment target, effect of the near-point displacement (ยง7.3), macOS packaging and ad-hoc re-signing notes (ยง10). Co-Authored-By: Claude Opus 5.5 --- apps/easy_gui/README.md | 257 ++++++++++++++++++ apps/easy_gui/docs/DESIGN.md | 22 +- apps/easy_gui/docs/images/easy_gui_window.png | Bin 0 -> 17213 bytes 3 files changed, 278 insertions(+), 1 deletion(-) create mode 100644 apps/easy_gui/README.md create mode 100644 apps/easy_gui/docs/images/easy_gui_window.png diff --git a/apps/easy_gui/README.md b/apps/easy_gui/README.md new file mode 100644 index 00000000..ed4ecc6a --- /dev/null +++ b/apps/easy_gui/README.md @@ -0,0 +1,257 @@ +# HDMapping Easy Forestry + +Easy Forestry is a single window that turns the folder a Mandeye scanner records into a 3D point cloud (a `.laz` file) +and the path you walked (a `.csv` file). It uses the processing settings that HDMapping recommends for forests, so there +is nothing to tune. + +![The Easy Forestry window after choosing a recording](docs/images/easy_gui_window.png) + +- [Who it is for](#who-it-is-for) +- [What your computer needs](#what-your-computer-needs) +- [Installation](#installation) +- [How to use it](#how-to-use-it) +- [What you get](#what-you-get) +- [Troubleshooting](#troubleshooting) +- [Getting help](#getting-help) +- [Credits and license](#credits-and-license) + +## Who it is for + +Easy Forestry is for anyone who walks a forest plot with a Mandeye scanner and wants a point cloud without learning the +full HDMapping toolchain: forestry partners, students, field crews. It is designed for forests; it works elsewhere, but +its settings are tuned for forests. + +What it does **not** do: + +- **One walk at a time.** It processes one recording folder. It does not merge several walks into one cloud. +- **No real-world coordinates.** It does not use GNSS or ground control points. Coordinates are in metres, starting at + the spot where you started walking, with the vertical axis (Z) pointing up. +- **No loop closure yet.** If you return to your starting point, small drift is not corrected. (Planned for a later + version.) +- **One lidar only.** Scanners with two or more lidars are refused by the check. + +For any of these, use the full HDMapping apps (see the +[HDMapping wiki](https://github.com/MapsHD/HDMapping/wiki)). Easy Forestry keeps its intermediate results so that you +can continue there. + +## What your computer needs + +| | Mac | Windows | +|---|---|---| +| System | macOS 13 (Ventura) or newer | Coming soon, not yet tested | +| Processor | Apple silicon (M1 or newer). Intel Macs are not supported in v1. | 64-bit Intel or AMD processor | +| Memory (RAM) | 16 GB recommended | 16 GB recommended | + +**Memory:** a 10-minute walk needs about 9 GB of memory while it is being processed. Longer walks need more. Close +other large programs while Easy Forestry is working. + +**Disk space, rule of thumb:** while processing, Easy Forestry needs free space of about **20 times the size of the +recording's lidar files**, which is roughly **1.5 GB per minute of walking**. About a tenth of that stays as results; +the rest is freed when processing ends. You do not have to work this out yourself: the check (step 3 below) tells you +exactly how much is needed and how much is free. + +**Time:** on a recent laptop, processing takes about twice as long as the walk (a 10-minute walk takes about +20-25 minutes). + +## Installation + +Easy Forestry is free and needs no account. Your computer will warn you the first time you open it, because the program +is not signed by a company that Apple or Microsoft knows. The steps below show how to get past the warning; you only +need to do this once. + +### Mac + +1. Download **`HDMapping-EasyForestry-mac.dmg`** from *[download link โ€” coming with the first release]*. +2. Double-click the `.dmg` file. In the window that opens, drag **HDMapping Easy Forestry** onto the **Applications** + folder. +3. Open your **Applications** folder and double-click **HDMapping Easy Forestry**. +4. macOS says **"โ€œHDMapping Easy Forestryโ€ Not Opened"** (Apple could not verify it is free of malware). Click + **Done**. +5. Open **System Settings** โ†’ **Privacy & Security**. Scroll down to the **Security** section. Next to the line + **"โ€œHDMapping Easy Forestryโ€ was blocked to protect your Mac."** click **Open Anyway**. +6. Enter your Mac password (or use Touch ID), then click **Open Anyway** once more in the dialog that appears. + +From now on Easy Forestry opens normally. If macOS ever says the app **"is damaged and can't be opened"**, see +[Troubleshooting](#troubleshooting). + +### Windows (coming soon, not yet tested) + +The Windows version has not been built and tested yet. These steps are what installation will look like: + +1. Download **`HDMapping-EasyForestry-windows.zip`** from *[download link โ€” coming with the first release]*. +2. Before unpacking: right-click the downloaded `.zip` file โ†’ **Properties** โ†’ on the **General** tab, tick + **Unblock** at the bottom โ†’ **OK**. (If there is no Unblock box, skip this step.) +3. Right-click the `.zip` file โ†’ **Extract Allโ€ฆ** โ†’ choose a folder, for example `Documents\HDMapping Easy Forestry` + โ†’ **Extract**. Keep all files in that folder together: Easy Forestry needs the other files next to it. +4. Open the folder and double-click **`easy_gui.exe`**. +5. If a blue window says **"Windows protected your PC"**: click **More info**, then **Run anyway**. + +Tip: right-click `easy_gui.exe` โ†’ **Show more options** โ†’ **Send to** โ†’ **Desktop (create shortcut)** to start it +from your desktop next time. + +## How to use it + +The window guides you through four steps, top to bottom. + +**1. Recording.** Click **Choose recordingโ€ฆ** and select the folder the scanner wrote, for example +`continousScanning_0012`. It contains files named `lidar0000.laz`, `imu0000.csv` and so on. You can also drag the +folder onto the window. You can work straight from the scanner's USB stick: Easy Forestry never changes or adds +anything in the recording folder. Copying the folder to your computer first makes processing a little faster. + +**2. Output folder.** Easy Forestry suggests `Documents/HDMapping Easy Forestry/`. To use a +different place, click **Choose output folderโ€ฆ**. The folder must be new or empty. + +**3. Check.** Easy Forestry checks the recording and the free disk space automatically. Each line is marked: + +- **[OK]** (green): all good. +- **[!]** (amber): worth knowing, but you can go on. +- **[X]** (red): this must be fixed before you can start. See [Troubleshooting](#troubleshooting). + +**4. Process.** Click **Start processing**. Easy Forestry shows which stage it is in, a progress bar and the time left. +You can keep using your computer, but it will be busy. **Cancel** stops processing and deletes the unfinished results. +Closing the window also stops processing. + +When it is done, click **Open output folder** to see the results, or **Process another recording** to start again. + +**Advanced settings** (you normally do not need these): + +- **Closest points kept** (default 0.4 m): points closer to the scanner than this are left out. The default removes + the operator's body and the scanner handle. A lower value keeps more of the close surroundings, but may include you. +- **Place all points by their measurement time (experimental)**: an alternative way of placing the points. Tests showed + it slightly less sharp than the default, so leave it off unless you are comparing the two. +- **Keep intermediate files**: keeps extra working files, useful when reporting a problem. + +## What you get + +The output folder contains: + +| File | What it is | +|---|---| +| `point_cloud.laz` | The point cloud. Opens in CloudCompare, QGIS, LAStools, lidR and most other point cloud software. | +| `trajectory.csv` | The path of the scanner: where it was and which way it pointed, about 200 times a second. Opens in any spreadsheet program. | +| `README.txt` | A description of both files, with the numbers of this particular run. | +| `easy_gui.log` | A record of everything that happened during processing. Useful when asking for help. | +| `work/` | Intermediate results. `work/lio_session/session.mjs` can be opened in the full HDMapping apps to improve the result further or to give it real-world coordinates. | + +### The point cloud (`point_cloud.laz`) + +Every point has these values: + +| Field | Meaning | +|---|---| +| X, Y, Z | Position in metres. (0, 0, 0) is where you started walking; Z points up. | +| intensity | How strongly the surface reflected the laser. Bright, smooth surfaces give high values. | +| gps_time | When the point was measured, in seconds since 1 January 1970 (UTC). Despite the name, this is not GPS time. | +| point_source_id | Which roughly 5-second piece of the walk the point belongs to. | +| user_data | Which lidar measured the point (always the same, since Easy Forestry handles one lidar). | +| **range_m** | **Distance from the scanner to the point when it was measured, in metres.** Far points are less precise than near ones, so this is useful for keeping only points measured from close by, for example only points closer than 15 m for measuring stem diameters. | +| **point_origin** | **Where the point comes from:** | +| | **0** โ€” the main points, placed by HDMapping's motion tracking (lidar odometry). Most of the cloud. | +| | **1** โ€” close-range points. HDMapping's motion tracking ignores everything within 1 m of the scanner; Easy Forestry adds these points back afterwards (from the "Closest points kept" distance outwards), so the ground and stems right next to your path are not missing. | +| | **2** โ€” filled-in points. HDMapping's motion tracking works in tenth-of-a-second pieces and skips a few milliseconds between them; Easy Forestry adds the points measured in those short gaps. | + +Points 1 and 2 line up with the main points typically within a millimetre, and in 99 % of cases within about 1 cm. +You can switch them off by filtering on `point_origin`, for example to see exactly what HDMapping alone would have +produced. + +### The trajectory (`trajectory.csv`) + +One row per scanner position, with a header row. The most useful columns: + +| Column | Meaning | +|---|---| +| timestamp_unix_s | Time, in the same seconds as the point cloud's gps_time. | +| x, y, z | Position of the scanner in metres, in the same coordinates as the point cloud. | +| qw, qx, qy, qz | Which way the scanner pointed, as a quaternion. | +| om_deg, fi_deg, ka_deg | The same direction as three angles in degrees. | +| scan_index | Matches point_source_id in the point cloud. | + +`README.txt` describes the remaining columns. + +### Colouring the point cloud by a field in CloudCompare + +[CloudCompare](https://www.cloudcompare.org) is free and opens `point_cloud.laz` directly. + +1. **File โ†’ Open**, choose `point_cloud.laz`. In the dialog that lists the fields, keep everything ticked and click + **Apply all**. +2. Click the point cloud in the **DB Tree** panel on the left. +3. In the **Properties** panel below it, find **Colors** and choose **Scalar field**. +4. Just below, under **Scalar Fields**, set **Active** to `range_m` or `point_origin` (or `Intensity`). +5. To keep only some points, for example only the main points: **Edit โ†’ Scalar fields โ†’ Filter By Value**, enter a + range (for example 0 to 0 for `point_origin`), and click **Export**. CloudCompare creates a new cloud with only + those points. + +## Troubleshooting + +The check (step 3) explains problems in plain words. Here is what to do about each one. + +**Problems with the recording** + +| Message (shortened) | What to do | +|---|---| +| "The recording folder โ€ฆ does not exist or is not a folder." | Choose the folder again; the USB stick may have been unplugged. | +| "This folder does not look like a Mandeye recording: there are no point cloud files / no IMU files." | You chose the wrong folder. Choose the folder that contains `lidarโ€ฆ.laz` and `imuโ€ฆ.csv` files, for example `continousScanning_0012`, not the folder above or below it. | +| "There are N point cloud files but M IMU files. The unmatched file(s) at the end will not be processed." | Just information. Usually the scanner stopped between two files; the last few seconds are left out. | +| "Some file numbers do not match." | A file is probably missing in the middle of the recording (for example after an incomplete copy). Copy the recording from the scanner again. If that does not help, the result may have a gap at that point. | +| "These point cloud files cannot be read (damaged, or not fully copied from the scanner)." | Copy the recording again. If the damaged files are the last ones, you can also delete them from your copy. | +| "This recording contains data from more than one lidar." / "โ€ฆdescribes more than one lidar." / "โ€ฆlists more than one lidar." | Easy Forestry handles single-lidar scanners only. Use the full HDMapping apps. | +| "The calibration file โ€ฆ could not be read and will be ignored." | Just information, processing continues without it. If your scanner needs a calibration file, ask whoever set up the scanner for a correct one. | +| "The calibration file โ€ฆ belongs to a different scanner." / "โ€ฆ does not name a valid IMU." | The calibration file in the folder does not match the scanner. Remove it from your copy of the recording, or replace it with the right one. | +| "The IMU files could not be read." | The motion-sensor files (`imuโ€ฆ.csv`) are damaged or empty. Copy the recording again. | +| "The IMU and point cloud timestamps do not overlap." | The recording is damaged; the two kinds of files do not belong together. Copy it again from the scanner. | +| "The recording is very short." | Just a warning. Processing needs some walking; recordings of less than about 20 seconds may give poor results. | + +**Problems with the output folder** + +| Message (shortened) | What to do | +|---|---| +| "Choose an output folder." | Click **Choose output folderโ€ฆ**. | +| "The output folder is inside the recording folder." | Choose a folder somewhere else. Easy Forestry never writes into the recording. | +| "The output folder โ€ฆ already contains files." | Choose a new or empty folder, or empty this one first. | +| "Not enough free disk space โ€ฆ" | Free some space (the message says how much is needed), or choose an output folder on another disk with more space. | +| "Free disk space is tight." | You can start, but make sure nothing else fills the disk while Easy Forestry works. | +| "Could not determine the free disk space." | Usually harmless. Make sure the disk has plenty of space (see the rule of thumb above). | + +**Problems during processing** + +| Message | What to do | +|---|---| +| "Easy Forestry could not find the lidar odometry program โ€ฆ" | Some files of the installation are missing. Install Easy Forestry again and keep all its files together. | +| "Lidar odometry stopped unexpectedly." | Most often the computer ran out of memory. Close other programs and try again. If it happens again, see [Getting help](#getting-help). | +| "Could not create the output folder โ€ฆ" / "Could not write โ€ฆ" | The disk may be full, or you may not be allowed to write there. Choose another output folder. | + +**Mac: "โ€œHDMapping Easy Forestryโ€ is damaged and can't be opened."** This sometimes happens with downloaded apps that +are not signed. Open the **Terminal** app, paste the following line and press Enter, then open Easy Forestry again: + +``` +xattr -dr com.apple.quarantine "/Applications/HDMapping Easy Forestry.app" +``` + +## Getting help + +Open an issue at and attach `easy_gui.log` from the +output folder. Describe what you did and which message you saw. + +## Credits and license + +Easy Forestry is built on [HDMapping](https://github.com/MapsHD/HDMapping) by Janusz Bฤ™dkowski and the MapsHD +contributors. All the processing (motion tracking and point placement) is HDMapping's; Easy Forestry adds the window, +the checks and the export. Easy Forestry and HDMapping are released under the [MIT License](../../LICENSE). + +If you use the results in your work, please cite HDMapping: + +```bibtex +@article{BEDKOWSKI2024101618, + title = {Open source, open hardware hand-held mobile mapping system for large scale surveys}, + journal = {SoftwareX}, + volume = {25}, + pages = {101618}, + year = {2024}, + issn = {2352-7110}, + doi = {https://doi.org/10.1016/j.softx.2023.101618}, + url = {https://www.sciencedirect.com/science/article/pii/S235271102300314X}, + author = {Janusz Bฤ™dkowski} +} +``` + +How Easy Forestry works and how its results were checked is described in [docs/DESIGN.md](docs/DESIGN.md). diff --git a/apps/easy_gui/docs/DESIGN.md b/apps/easy_gui/docs/DESIGN.md index 05e0a764..5a9bccf7 100644 --- a/apps/easy_gui/docs/DESIGN.md +++ b/apps/easy_gui/docs/DESIGN.md @@ -3,6 +3,9 @@ This document explains how Easy GUI works, why it is built the way it is, and how its output was validated. It is for collaborators and reviewers who know HDMapping. The user guide is [../README.md](../README.md). +Naming: the user-facing name is **HDMapping Easy Forestry** (window title, user guide, release packages). Internally, +the code keeps the name Easy GUI (`apps/easy_gui`, the `easy_gui` binary and targets), and so does this document. + Contents: 1. [Goal and scope](#1-goal-and-scope) @@ -81,6 +84,11 @@ native message box on `bad_alloc`), and a real cancel: killing the process group executable, under the plain name or the `HDMAPPING_BINARY_PREFIX` name; `HDMAPPING_STEP1_EXECUTABLE` overrides the lookup. +**Building only Easy GUI:** `cmake -B build -DHDMAPPING_EASY_GUI_ONLY=ON` builds and installs only `easy_gui`, +`lidar_odometry_step_1` and the libraries they link (plus `easy_gui_tests` with `BUILD_TESTING=ON`). The default is +off, which leaves the upstream build unchanged. On macOS this mode also sets `CMAKE_OSX_DEPLOYMENT_TARGET` to 13.0 +(unless given), the lowest version the code builds for; without it the minimum is the build machine's macOS version. + Quirks of headless step 1 (`lidar_odometry_step_1 `) that Easy GUI handles: - The exit code is 0 even after a caught exception, so success is judged by the presence of `session.mjs`. @@ -326,7 +334,9 @@ placed the near-point way and flagged `point_origin = 3`. Their nearest-neighbou Rebuilding step 1's ramp from the raw files for scans 40โ€“49 reproduces all 7,595,561 step 1 points (โ‰ค 0.17 mm) and shows that the whole difference comes from step 1's ramp time choosing a different node than the real time would: -|ramp โˆ’ raw| p50 1.2 ms, p99 11.6 ms, max 17 ms, giving displacements of p90 9 mm, max 56 mm. +|ramp โˆ’ raw| p50 1.2 ms, p99 11.6 ms, max 17 ms, giving displacements of p90 9 mm, max 56 mm. Displacements of this +size have a negligible effect on sharpness and stem diameters: placing all points by raw time moves them by the same +mechanism and changes ground and stem thickness by less than 1 mm and diameters by a median of 0.4 mm (ยง8). ### 7.4 Gap points sit where a missing slice of step 1's points would @@ -393,6 +403,16 @@ always reports the breakdown, warns below 1.5ร— the estimate and refuses below 1 - **Windows:** `child_process.cpp` has a Windows implementation (CreateProcess + job object) that has not been compiled yet. - **v2:** gated pose-graph loop closure (ยง1). +- **Packaging (macOS, not done yet):** + - Bundle `easy_gui` and `lidar_odometry_step_1` in one `HDMapping Easy Forestry.app` (step 1 must stay next to + `easy_gui`), with their dylibs, and rewrite the library paths. + - Then re-sign the whole bundle ad hoc, because rewriting library paths invalidates the signatures and Apple silicon + refuses to run unsigned code: `codesign --force --deep -s - "HDMapping Easy Forestry.app"`. This must include the + bundled step 1 binary and every bundled dylib. + - Every bundled dylib must be built for macOS 13 too, or the app's real minimum is higher than the README says. A + build against Homebrew links `libproj` (into both programs) and `libhwloc` (through TBB), both built for the build + machine's macOS (26 here; the linker warns). Build them from source with `MACOSX_DEPLOYMENT_TARGET=13.0`, or drop + them from Easy GUI's link, and check every file with `vtool -show-build` (`minos`). Tests: `easy_gui_tests` (doctest, synthetic recordings and sessions). The integration tests against the real step 1 binary run only when `EASY_GUI_TEST_RECORDING=` is set; a folder of symlinks to the first 4 file diff --git a/apps/easy_gui/docs/images/easy_gui_window.png b/apps/easy_gui/docs/images/easy_gui_window.png new file mode 100644 index 0000000000000000000000000000000000000000..95fde85c8d31b5c87166df8b10319f1f45927624 GIT binary patch literal 17213 zcmdUXd03L^_cvvWGg;O+-%87+)tPCrG|Lq#n>L#)oh)&yaVv1i2u%=h#>)EUQkj~j zl9d)Ik}0Ah(yHc$8G?aGWob%4hCqP8`zVRd^u6Xc@B8~*?|Z$^A6<_R&%K@d+~<7G z=bWeW`}Xdb{jJ%zAP{KwuASRFL7jA@=}^?r!O3U^S!HwuHI5n{%Kgx|_LEzPxQU^|bEE=4pny?h*{=PSrh{XEjZCqwnbJ zC;zbPxJ6l6S=F_*%r6tsH*;ue{l4Y6`{nfX^erUkEB~UXuv8MCLJtaLU+78G?$d_9 zt0lYVAxC!v1Ufgq$m@q^90_f@n7+OuQh9)?o8_e~UzQupGd=BiaTl3X^r|lBY!Ym- zNTn*XpWWXR8NTLehYD_A9&uEg;!1ZjEg^+5M4oMsVyAX{XYd6<)d@F+I%7O?` zbj5QVtiAC3o*nA3-$D8XZkzEsGW2xbfq7AUn-z-|ypJ7w@95)!0pW7nFY022{q1|*^z_QrPQI3tQ2R>++25B$F7h9y z>SSs)-UJY?-|aF}H(H3HWVUnjhZ1K?r>LS+)Z50yzK{=cCOMgi2v9x^BGT-W+MI?^FBymQB&9zW% z3lMvV9)4esUL>U3Ec;nI&tt@i(Z-6)p2k`*Na<%{55cvVg1F@w!agUu zr@a$AMNkNzG3&7zCbQ=Hbb(w}hgXO1mU5*PF+WYBS7i5NQIAz*C5f{V=kYESeJ`hN{_I zDNQPP?k9NSDexqyN;>G<*2@)S+qR;%+X3vXvW7Ay$~A_{63=Ak`;;-&VFLC|PeD8} zxrL0K6F7+Q6Y%LRnEXQ0wZgvV1+qhf*x2x*BLaNX6wvsg0#28Y=$nr$vgabJ4E()f zf3NGZ^xFK**TVefV5XK)%+3&7VtY~71R@Smiu#or`a?u=d7+Xfm4?8m!=DrdS#j;k znztGgxx2bYTw`;Ts)edoSW3)6dc|!-!RE?5o!88AEn_yDc;>(TxAm%7E{dwoXeQU` zX#7oWU-9{n5h6NwKz?8NTU|KZ|Ah7%aO_bJ!F~&T z%lT$uSb`$pjhg6u81qoN#o6Exq+%7QqJMZe<~#l7*aOzYvvmFv(537t4`6UN1JHiG zIWT&3YU}F!xsM$TShjrPs>7HqTGIs})-Q6KNefp+=H~ILcVGVh%Ps!b9#?P7^gbAP z;&ePNF1xw8`8SfmnCYFcywqzikJtNL&dtr;5a*|=h^eJL*EyF7r*vt{)dOL3ieBuY zc-vX(CZKn#fQf#wx%FXy?KkcrJ1v?VO31td>_RHNKxm7EAfk-NMw?JoXrq;jE6r7~ zCBQE?tTYGfbL{NfGHn2wS|&z+P?-;E3rx=;$nLuk49Hl^>#kmJ!p6M-FxImg*f&LG zEzKBl@kYzBH#RfJ{qEAllJLTnuRKjrY?V2BF`05R ze@%;SkO?<)XUkQC?BhwA9#m0x-JVB0x~1>6d$O3qHfomUFAG2XqfTcBuWs46ulc*N z5xYuXyx{H{cjlW%p8TMjh#os#hi|f<5?l#Qq%tjZNJ(!B0ExcO+OuV>!Ni3A23hI% zS-+S1VZMIyzX#g%T`v^%b<7-9o#l>>jstgJm!_c)4oJFq45nO6rcs>QDmLBglpYzB zl9)~vQfbLB#;`frj{WW`-Jox%eW;_^>4?QE6g&Vnlhw|ETiUqe@d`t|F96iSbU1i$ z2@QOHziQWIfjT*&|I{bg0gvRSw;+`N%p?mjQ-GH~#~Ah{3Lfvu811hNol9)FP-c3q zk1SZjrR{CD;>w~+G3r>1!GummgwC~SI{E3X4&qF5J{ns?JFnT5ixu1#L6mE#csKaN zPNy~kc}bE*e`L{7p;=r|8l^M3lC-g@AZ9Jk!iyUsiry&W`)1?Z(-<8Mo8-}Lw7NY+ zwVl-Ab}9oZIBY|9tWQnvDZNJ2db$lD2`sgLF}syxpK9FsgoD8o*aMv2nqg^-D8^?{ z(gal&NTH`2EIbBOJWg+UmWo%{M#^HClk4z@1~oOZs_YukS}EobQNdRB=N8hlev&$d z!Bki77W%^SDW5QzW{>}LESGw0WMSV#e@HKKcXf0b-c{)7QL-2FTM@Hr0GYeyfXT_y zn1K-R>WuT=gATs@Qns{kIQ=4mXhRH0gFbt5RZ!!2FtpyAw8K55hm7TwxLWQ`X$T;_ zaQBabh1~!guX4$Sk;^+tY8$!C+M|yZ19Y_7}C5kx!u*=NhW=5w%7bdJsvR|9#X=%My2yTje3d9`sZM$pq}d z;I3sMCe7J3@nd`XLSpAn1Y~XlxS#`$W`TRi$>rX%O_i=_Q+85n)CWaI@P+o(E1B%kr6r*$jU zk)cghCDQ630p5$ERNT#A7}Mh0jtMn~392k8{x(|GQz4C+qPIea=!?R_VoNW-ee=c- z4%%NnZJo!ID-H!No%SNLzl(aw}f& zcT*Dt#;+tRL6g$_7098xtf3DLiEwvj>~}i^a}`p8740n9aX=Phi^uPK1?p5NIoZl z?;d=gIpqFr*vS>ofa{L1A(A4KF$1_xT8tX|KE5GxQ;cRWZ_IJ&t>DG^wuU6Tneq^* zxiS{iXq0``%WmU9F<-7$bqP~je_eyj9UX5%BcV-JE*tHD5c73=P82bG6I=0%Fo~y@ zb>ugMUe9!*7uehPn^_lDrnTH)-r41Sw~xnEr@E4=xgn)S#18vKzLqwHswl!x2fCHU zi$;g9ENIz0I)Yva8gpQCQ|&84Uw1*15D{7RFxxqZJvUVZb(h zodMiXcKPK>1LK`R+4){@lgzemXbsCE#O^dasAkzbs zL|F_XFVL)Y=ZbXO)>NYX#u#d0Dn%}9`F&4u0r^cTk1vvu8BqcWNv^09OQ|>jMR=ZF zGTO6V2FcN5RLx~SzuYubqTEF~k7zhRtv@UXs8&_9oe3|V&+ppfmK`(N=pO%1;q&NG z3s&Tu8U1N8`k8zGApk5`-sriFF!=24^;V-VOcvbc;BTky_Z^FwCw%2sYz}ag_YODl zZj+aewx8U?|78Y92e|(31dq%F;MC%nxHv^64&vyTjf8mdwMzAFOY5L{{kyRrCt?&ponFMJJTz(v?k4W1 zh_wx&J1u-~4(&^+Eo<1Xba3y)E{4v!A46;|v!4Z^iMnVu*JTWAPY^>(6F)lKzD)^p zZ3MvR>Wngqy&LxkAtx0fXP0GXyR&OtVsM3N<)JAp)$B1F1A71b%IEBH2J4*Bqt0|k z!(R9Ti@TS;8Jpg=+2f>~t>H4n+z$8FU1QHDbU%Dm;>R~{T;PnwB^hz2)Ebq{+Tb!V zgd@9G_m|c0Ph8g;yuENJC!Csn8t^@Q!EKSa9h+2#CG`5$@cW$V%7iab&N`+~ug800 z$MpC{+k%10?dCd}CFSM!?A2S7H`d$#>z&;2u4v`q1ASs05oiw7I)&rjs5eF;)HYh46bUm`v;`ghcRhL_%(KRkBV1i z-ZfLSSmyocx9Cbth_*^C-U6d)ZD9JYrqiga%zUh0PUR=th4xt>c%_lDJVCdXi1q zHsgpH;rGmCc4o#QJ6rR%*blqF*)fZ$^fEQO>6BxP%ZJE76z=7r*^dcg^c_f1o17pa zd287@#>CY0LS@8z>5XA&p$xTA9-WyZ_9A5!DsduBq#CuMGW0d49yu`NDFB}ccV-RB zS)yiNs^uLFpQFMk+M_*{l4ApLRAw!~-ZC?XLX@Ei*wy4V8?|FtDFf%uPuipI-j7l* z_Vv2QLf+^Ub2FiVFcMzG=*Cw=67|gK(!_Ps@gE1t-W<>Dm4(V9rPgoyOOZZ;xS zY#MKe2tD=G8u_K5ezXmn3zZYelxwPy-K|O04ekQD7QslW?iYG1X~Rx@j#>m2lib7R zx4Bvt^#rnO`2K^4!EQmbr*J^iqRv*?O&CsYqg8Hk=BuupZ38aL;KY8wPrggRJPI!5 zchrx-14~W0c%^|;-8ZQC!2dY$QGyPpA_U1e5iBzVF5lpes6FJJA=SCnXR6F zmdsBo!KjC;4dh`W#>9?$+9V$5M@d=AUp|r9F}_>{QT+_64%TEF3kU?eVHYBswIz80 zd%e2ng(4fb8P5I}o{te7H-D_oj-IVI{ueje86w_BJ-EHEZ`Xjju$-@vFW+@H*de4K z`6o486LADQ%vq~~l^ylyHR^qe+(Rb~jKDCdkK-RA3z?^EFn;a~8@zK#+Cm!!7}LN{s(YWxqT?2P9F0GX5e zD>pId0l+;weB?Y2=5tDo4Q7eW>d$T3=Q47I>?uuwL00^?ki z0;KyjAcs+Y{AlYV@5*a}5m;x!>I_BP2kh<2AY$+8P7PH{A0C=*&PJq^7YMi3+`VL5 z5%aXbEF=hyxYE{gRj{jvAU+8E%hNdix+=1i^OGC*UAroc7`;g*4T3i?d!pvI-Quuq zR7&|i92?d^5C)>vf{@xP0fBQmYd@?=ypbUgdiD;YjElY&-Ca4vx{zgN>=Y;mBx*6vvbH302qw&J0Z`R zsw~uZs;=ZzPvO0j=t>Tr$o78r#QIa3Ybl|@Q95!qG=`aP>;~_?Rc022kn)+rvy}or z2`OXGgN}Z-d%dPR+oQ0Qf{?orRF}L1!*?F{3ryO>Ol0*@>346^@G$GNlJ|8czw6B*In6(AjnfPljIvP&wAaLY< z44{g-a*o;F?0OV$RUCgA5iZOxv|F;f6J#53$gS@gtN(Ea`Rt)eEsFF@RVMPdP<_}h z+js@cgjM?E2PIqX1amc*h-G#PjUMsLFe=*%sjXG+x5?im<+C{)gTBRVa(}iW`v#fr zsW{IhH0)aB@x8nQN3;x!k{ppc$U_{iva7(u)H#DQO zR~-#LXlhIDynU`-fFJ6M8tA?16aYk1g6uSM|MRYBC#3mmjJf;G?A`8+o1XSd zvz!t-2flV{W+s1R5uqryE0I;T0Vo^|R`mPYCa4$~LU6n3bgBol1~0 ze*`dV`nZoCwT_!n#Ejo%C$WHKV0k=nnF%^KN(PKxn@3TfZCJ$RQ;!zj%RjbY*!ggs z!Nl$Yfas&vRa1}#n`Q;O1cK-G(}>0_JK-PLnLt)#Lesv+WB2pTPLsMZ&{aD4If2=~ zMr{AY=H83jWK4VI@6r8yw{GI?=MP%OFC8bE6PAxhMS4p09~s#D=RvRCjCt*Yx=t^q zOf!sfmDS~fhwfzRxX6TXtIekrgKz~N_t9A))(YuwA=(xkXOA0wxCP%|7^dky_W}Tq zV;vk~wyualEqi8(`y7i9ly}x>N6hJrME%BXQn(_u29G|;hOhy@D<-Q znhnJQx4d6&@%AO!_yiJ}!LlW4XmTyb(#-yQ%Rsk=&MsjxP^JJ}uK;rQBXhu}&wyfq zS+(zTm+AyoyCq+nlr?NL_JXeWxX-U+-%_yZL`8;icN398``7Z3X1eBtMnPsvzhq#8 zL#8}}L!{}DOMGn}zmeEr5myX;tzPh{+9u%bOvuPV*DnSFiK0*CO9*`Z%TDt!5QzF$yEhhe>(Q4s&f#) z&pL4|78h6;D__Ufx-oPi&1UbbTRw9HQEw%I%H3^ZNaJn#afl7B*Y_AVEAoE@n(Oq-*=G#q~8a`bT;a;|FKSKXo+oTUcRt1eRXaq)!x0IHa4(r4Z8r}6Q`VcW*`BLRbmv04Y3rTOu zY&zaP#33RAr?Rc-kD72`3w(H-_b7*78SW zu~OaX2K{+rzIeG{R@0_^0o zsivp1NOhS}BQIjdRe1HQ0*-ThZ>uE?sgpqC_VL60;6x;Bs+Aqvth-QVHTI((Fawrq zGaaByMxXwjvAGp6`w=ky)>U8hL5~an&+O9IA?epo{zH)ZPfQ-r(Wb#qpKO7unp9m` z&;+xx>+-dihl_YzAjuP5J0^%f+3W#3#PPsBUI8xlKl~^hSn)#C)$+hC`-~fdE{pME z=<;$CA&f0<#%qAU@-hAd5D|!w!NPK*$H^^My0fx3RkDr@n4L$2&S`T6>m&%f_8laB z9DoCfM`GvlZj_LUAy4ZsRov;b8k^l@*6ZT0B>$^>ZRhKnPlP7#>z@PI0!1KPIKtwHXpEVCc}Wdz4S-CQ z@tZV96bdjhZ(Ul6`@%3_NWHu+Z8_A`wWpTXQil;E)HQCZm+~5(mY=o%YPQ|JrCztr zkt_MqICC?|#X0zJ%bX>CX!EPeCz^I)i@a)Y4ldm;7Hx_T>Q^z5k6Z>3{!sJ`AEKg; ziK>(d@J2g86L8Fm+iKO)*zypQjd8R4t|2n(Og>2ZcE!aEmmvr{hPC66cEtH4xi~A@}S@ ze~3<{S%#}?>Xk=?)%f@ z@S`iA_zCZmhdSJ6696`6aK!y4`Byu{>~d)K396u}T~5j<9)Bacl@^ zKD9;?H6UB;1V%k;%*~00!t2=3tlfG1ZaLn|Zgz-g1OnqTs6mSu-_+Yr;(0F$L+1vB zjmlw~m!rXv4=PlL9jt24orhZ&NvNHYmr9d&H0I@EOs*Sg2di*nAKvWzI=%}m!qpd5 z*Su38sC$4ko19IEu``T?T!a6E9$8Dhw^d8T_{7apv%-Hs*@kNFP<`8BUTEhUEa zQHKVcW>3I% zVu?GE_1;-_3hl#akt5^VOr5|DZSjKA%?GxJY)njFQ>A5{4-dLyQAg{11KY(SH&$mV zYnJd1Z;)gnLbpJYAA8doKOw<=x5bcUlgfEs*N1-e51VE|Rf`0&kncl_tii_f9u_@G z{{bj86r3tSf@v;WPGaJN*sCO^Ee;;*dl4VC zD(D5~J~B&K;!dt_Z5On7_g#vkMvz(O$$a_M9qnSd4?K?&g!}zBAs_yEvBh}B(Pp5` zs%3kAX{sG5HSpRDS>hx-~e;W8A|yEuF@NO7w@QSc>Pek_<)GZV%gETuFr9P*L0grLw*0 zbFNbWtn~s%fOH~GkAAQ9t)|=ZdDiSIlCzATB;yuHOnsM~6sht#>P>8y`)gC+3<9|$ zNksA99xsfmU}B)fg2(|0EF^ECQ)C*gu4F}G5J7ATB=P|kgsjB)!{sK0?AwJRG>V`Y ztuc_P$zppBP>#?nJvd&o2cV*3H@CYJ_T&ZeDZdu&Aad-9cz%~WyGG0LILWeL2jP!q zbHwEQ?oxZOFE4NvnFVuoJb=pcg%iuF8HiN^DMw6diIqQFL*cgf=Av-Eg1XyG46j=) z0^dqgJtTJ?ig*VmvXPWdX<m7LBy+WzI$hm zF_}#w#*@}dC0)?|OA`r_tsU&BdtrxzYsrS6ISiXT*MZ%E>ZXls1(be-VJoTbY_Vp~ zlVE<4KjRdNRSVQTVQ5)pzG)Z+7YOPLM6YO_w(IzRMZtSz`B-Zt?iF{;KrweCFfNx8zv)+cL#skF`1CQcKnr z{J~*#7(hTjJ@xX9npu!MT{4ADf5$0g^?xL=>D55S;kUM-=gpsCNbL6UJ-gUrXGqr! z?MQE2-g3z!Gb42~rIOymMewmP+B)&3#-zxDs*yS7;yN{uUnZtq3+PjI#rdM^NSHhb zGnLjN_wIY@%@$PWA9b$eQ)J6wQ4${J1fE^S0m>c4fh{-FZ)6)6vdXLRjiPs&r~Mho z5{Vo8z+eeb4jRbh$bX;(0{(H6%xPncv)nbpkB?SpP^TRZK_(a|6 z7g5KC&#&*@O-+c4$UBUdFKg(@GBWQc@`3qH3JU<3`V=3$_Z|O(=-snWkHfbG{HvDE zJ(uPg*4r!n(QA=dHwuCeSm()O?VRIs>BF;AKu5r(PnH47CF^u>yz+Y$l~-nZAqjT( zN?Uxm+Tpe+?5I{4des7kcN62nRjKA-#LoE~xzbZ|6iCRx7@iHjHU617NklnXAZYGT zAB6h>be0!oT1rMF$TFOyVenrm#@;*Y_ScnZ!6JA+Nt7D4j1k*6M zkRC)4sFj=S?FPH#c&a6e4rT%~lcxodaEP$CmG>?FDg$B8uW(3Pb^dVR-R*~%ppf5V zvf*_U?_OdIgb59-ydU5jSm~JNXL%J-dk7%j{{+}}6}FJQ;n(kUUcO6?hPw86!#75d zH^Rgc8b#6G{JA0^3V|VDut6kafOQk952{e`I^VB4F|pS|F~7Yu5pnOPWdEKWNXU&d zH6{GQkdx(jDO4NrdZe!o93Fu=CIyPBp zSq{24A3*X4D8NQcLPuYF{eKiNjMH?R^?(9N^-<0qH9LHnI4%D3Q=h%pWtxTkK-jtK z#EBE0Z;=SgnF6zeZH)N#wTEjB1b#LpK?}p~wZwOYVE4cqytxb|I|e;qSt8M&){BKe z&EEP%vD31TUBcKvip|Y+c01aa-3T}x6ZA`O$Y8o390E~_k_xh4yQj!G9(@+7TfBl8 zpr*=A4pf+PJS^5qZv_d|k+gcX)xt2_@a_dw_@fAaN13F18ttR*wczHm6iGCH8`K29*=tAGNE1Unn&b#V{et}{7H>Fni$Wx$FymyIDuJ6ZHrpbGfxkJ zg6D4Eez+;5#YnC4v z2G9)6-Y_J~VrC(uID>N)E|8N+c(SYV=y+8e-oW;!$02KkZ?Wx2j(j@T zXC_+Fb8;2mWB7g~;7Q?B@VP$?f9qxPQ*vc^Lzn0$ z-w9?nhkpA$R(`2;)xA;%B?ulGrA8#kjzf-OUiQ>>aTH_}ohn1Jq)(((Kcd9Aukff$}wW zG=ROo*{z;G?8Fn zCr+W1aXW}h&tkkT1imE5-bD0ofLVr_wT|RK(Ii=a4Vmci#$3POA7k`+UrEUlgMBlW z?hl%hZ2h1nVdxI;{XNU!RrS*kn{TC0K{&h8^7PHN&3<&H`7L$h!R>qzKuC%sH+-+eLC9Z#Y^ln)O9&K9`z z+5oJn=w{19)`#Zr!&kh}m0eCSXF%PCIdgHiefmA7VsVp0cdUjM4v=SsoYb*iCc~an zwLpn{xy9S3PviQUDw><@mNjT-6d`m_T-`tFlt7>de=d0nUr}Bv?QMMkjZxAJYWs;zSH5Szl2 zY@32}3wO6R0O1*HA;A2TTV{^UV1meNv|4`c&h#qM#@@P?8V=X|uDK6ttU_?I^GsY< z+;pq?QFdsM+)nx#CXRg{<4zu{c$@_I#_2lHe{*LieCDFfXSsD|z_*(VLW7BqdBU{T@WnO! z)xS1JzgMKVlDsPyIR~5t4A@Wx$Ko#wHQMUwV0S!xd%ytO@-DJU%ugd;IW9EnSHm=)^F?- z>8A=l6CmqD!Og*{8es9nbBL4)yevg&vVm)|*S7HS|H;&HiF%hnuk7m&=U{!#tsvBl8=M8-17SPmH2 zoWky%pBgMRyl&KoOSydX61*dBZB(tofTpwJAfF=%>vmx^G{`lGl4ksl6(nvZHdsDW z&}PQTdJ_Ho{R<0Q^8FDZ9jh}4b|-Oc5`?+;(v+uDd&?;j7{)?qaXW5IPv=f~^j*G_ z`f2I91*3m!x?prObru=alr(g%R zLFh6RwAt_SE}fwVcCPs|0Y7qnL?a&A9_cD;^p$=ire9=WhL%D;zorlB>T^MjO;b;^ z^GD9e`_DJ_X857jd-wu2WTUQ8A%gotVpM$ZAXAN5a>mRphG*7dn z0DC=pjpek(@%@h#LVo;D^x?Nb)zU6tirPU5E*%%APAN*2>ho~1t1=jlt zV~(|U$y~1NM}6RqONCg6Vp%WP@f9a)KN0Nd~j|!Hxq%(a6yX| zKz8`zXaZSYO&VOI>$dS*gVQ$Z-c1rhnCRVzW)_g|#bt@#yag;h zgN_?`7gZ!ROLVil4lp1MtUE8(Vp14o>f8J=jR2H0)}}5;f{yAmjo0bgz`rJX8%>of zhRfwZZ)f5)?t#zXBphT-=x^^D%_6DU``zI5**s0Vz&EO)8om+u7Ge<>=am)(e3BJI z1qAUcJxi3dAt&1RENUj5lG009ncVn&@{cz4`_vqt<6!iH2ya2e0t=?C687kIipMaZh z3Suy7$K*1BM{Et3*cv_8eo+s7^!c?LZMN1!4N-?^CXo Date: Sun, 18 Oct 2026 12:12:40 +0100 Subject: [PATCH 09/23] README: add a banner pointing to HDMapping Easy Forestry Co-Authored-By: Claude Opus 5.5 --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index b8e7817f..750fcd79 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,7 @@ +> **New: HDMapping Easy Forestry.** One window that turns a raw Mandeye recording folder into a point cloud (LAZ) and a +> trajectory (CSV) with the precision forestry settings, no other steps needed. See +> [apps/easy_gui/README.md](apps/easy_gui/README.md). + # Open-source (Windows, Linux and macOS), open hardware hand-held/wearable/mountable mobile mapping system for large-scale surveys: This is open-source software for large-scale 3D mapping using an open-hardware hand-held/wearable/mountable measurement device available at https://github.com/JanuszBedkowski/mandeye_controller. We provide an end-to-end mobile mapping framework that does not require any installation, including: From 281fa81872b778306b64cfc008879ff5eb8667d1 Mon Sep 17 00:00:00 2001 From: Martin Mokros Date: Sun, 18 Oct 2026 12:25:23 +0100 Subject: [PATCH 10/23] Rename user-facing name to Easy Forest HDMapping Window title and heading, messages, the generated README.txt, the LAS system identifier, the default output folder (Documents/Easy Forest HDMapping), the user README, the root README banner, DESIGN.md and the placeholder release names (EasyForestHDMapping-windows.zip, EasyForestHDMapping-mac.dmg, "Easy Forest HDMapping.app"). Screenshot retaken. Internal names (apps/easy_gui, easy_gui) are unchanged. Co-Authored-By: Claude Opus 5.5 --- CMakeLists.txt | 4 +- README.md | 2 +- apps/easy_gui/README.md | 70 +++++++++--------- apps/easy_gui/docs/DESIGN.md | 6 +- apps/easy_gui/docs/images/easy_gui_window.png | Bin 17213 -> 17165 bytes apps/easy_gui/src/easy_gui_main.cpp | 10 +-- apps/easy_gui/src/export.cpp | 8 +- apps/easy_gui/src/las14_writer.cpp | 2 +- apps/easy_gui/src/pipeline.cpp | 6 +- apps/easy_gui/src/preflight.cpp | 8 +- 10 files changed, 58 insertions(+), 58 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 277c431a..16c369c1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,7 +1,7 @@ cmake_minimum_required(VERSION 4.0.0) -# Easy Forestry release builds (HDMAPPING_EASY_GUI_ONLY) must run on older macOS versions than the build machine's, -# which is the default minimum. Must be set before project(). Pass -DCMAKE_OSX_DEPLOYMENT_TARGET to override. +# Easy Forest HDMapping release builds (HDMAPPING_EASY_GUI_ONLY) must run on older macOS versions than the +# build machine's, which is the default minimum. Must be set before project(). Pass -DCMAKE_OSX_DEPLOYMENT_TARGET to override. if(HDMAPPING_EASY_GUI_ONLY AND CMAKE_HOST_APPLE AND NOT CMAKE_OSX_DEPLOYMENT_TARGET) set(CMAKE_OSX_DEPLOYMENT_TARGET "13.0" CACHE STRING "Minimum macOS version" FORCE) endif() diff --git a/README.md b/README.md index 750fcd79..54542230 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -> **New: HDMapping Easy Forestry.** One window that turns a raw Mandeye recording folder into a point cloud (LAZ) and a +> **New: Easy Forest HDMapping.** One window that turns a raw Mandeye recording folder into a point cloud (LAZ) and a > trajectory (CSV) with the precision forestry settings, no other steps needed. See > [apps/easy_gui/README.md](apps/easy_gui/README.md). diff --git a/apps/easy_gui/README.md b/apps/easy_gui/README.md index ed4ecc6a..3312a92e 100644 --- a/apps/easy_gui/README.md +++ b/apps/easy_gui/README.md @@ -1,10 +1,10 @@ -# HDMapping Easy Forestry +# Easy Forest HDMapping -Easy Forestry is a single window that turns the folder a Mandeye scanner records into a 3D point cloud (a `.laz` file) +Easy Forest HDMapping is a single window that turns the folder a Mandeye scanner records into a 3D point cloud (a `.laz` file) and the path you walked (a `.csv` file). It uses the processing settings that HDMapping recommends for forests, so there is nothing to tune. -![The Easy Forestry window after choosing a recording](docs/images/easy_gui_window.png) +![The Easy Forest HDMapping window after choosing a recording](docs/images/easy_gui_window.png) - [Who it is for](#who-it-is-for) - [What your computer needs](#what-your-computer-needs) @@ -17,7 +17,7 @@ is nothing to tune. ## Who it is for -Easy Forestry is for anyone who walks a forest plot with a Mandeye scanner and wants a point cloud without learning the +Easy Forest HDMapping is for anyone who walks a forest plot with a Mandeye scanner and wants a point cloud without learning the full HDMapping toolchain: forestry partners, students, field crews. It is designed for forests; it works elsewhere, but its settings are tuned for forests. @@ -31,7 +31,7 @@ What it does **not** do: - **One lidar only.** Scanners with two or more lidars are refused by the check. For any of these, use the full HDMapping apps (see the -[HDMapping wiki](https://github.com/MapsHD/HDMapping/wiki)). Easy Forestry keeps its intermediate results so that you +[HDMapping wiki](https://github.com/MapsHD/HDMapping/wiki)). Easy Forest HDMapping keeps its intermediate results so that you can continue there. ## What your computer needs @@ -43,9 +43,9 @@ can continue there. | Memory (RAM) | 16 GB recommended | 16 GB recommended | **Memory:** a 10-minute walk needs about 9 GB of memory while it is being processed. Longer walks need more. Close -other large programs while Easy Forestry is working. +other large programs while Easy Forest HDMapping is working. -**Disk space, rule of thumb:** while processing, Easy Forestry needs free space of about **20 times the size of the +**Disk space, rule of thumb:** while processing, Easy Forest HDMapping needs free space of about **20 times the size of the recording's lidar files**, which is roughly **1.5 GB per minute of walking**. About a tenth of that stays as results; the rest is freed when processing ends. You do not have to work this out yourself: the check (step 3 below) tells you exactly how much is needed and how much is free. @@ -55,34 +55,34 @@ exactly how much is needed and how much is free. ## Installation -Easy Forestry is free and needs no account. Your computer will warn you the first time you open it, because the program +Easy Forest HDMapping is free and needs no account. Your computer will warn you the first time you open it, because the program is not signed by a company that Apple or Microsoft knows. The steps below show how to get past the warning; you only need to do this once. ### Mac -1. Download **`HDMapping-EasyForestry-mac.dmg`** from *[download link โ€” coming with the first release]*. -2. Double-click the `.dmg` file. In the window that opens, drag **HDMapping Easy Forestry** onto the **Applications** +1. Download **`EasyForestHDMapping-mac.dmg`** from *[download link โ€” coming with the first release]*. +2. Double-click the `.dmg` file. In the window that opens, drag **Easy Forest HDMapping** onto the **Applications** folder. -3. Open your **Applications** folder and double-click **HDMapping Easy Forestry**. -4. macOS says **"โ€œHDMapping Easy Forestryโ€ Not Opened"** (Apple could not verify it is free of malware). Click +3. Open your **Applications** folder and double-click **Easy Forest HDMapping**. +4. macOS says **"โ€œEasy Forest HDMappingโ€ Not Opened"** (Apple could not verify it is free of malware). Click **Done**. 5. Open **System Settings** โ†’ **Privacy & Security**. Scroll down to the **Security** section. Next to the line - **"โ€œHDMapping Easy Forestryโ€ was blocked to protect your Mac."** click **Open Anyway**. + **"โ€œEasy Forest HDMappingโ€ was blocked to protect your Mac."** click **Open Anyway**. 6. Enter your Mac password (or use Touch ID), then click **Open Anyway** once more in the dialog that appears. -From now on Easy Forestry opens normally. If macOS ever says the app **"is damaged and can't be opened"**, see +From now on Easy Forest HDMapping opens normally. If macOS ever says the app **"is damaged and can't be opened"**, see [Troubleshooting](#troubleshooting). ### Windows (coming soon, not yet tested) The Windows version has not been built and tested yet. These steps are what installation will look like: -1. Download **`HDMapping-EasyForestry-windows.zip`** from *[download link โ€” coming with the first release]*. +1. Download **`EasyForestHDMapping-windows.zip`** from *[download link โ€” coming with the first release]*. 2. Before unpacking: right-click the downloaded `.zip` file โ†’ **Properties** โ†’ on the **General** tab, tick **Unblock** at the bottom โ†’ **OK**. (If there is no Unblock box, skip this step.) -3. Right-click the `.zip` file โ†’ **Extract Allโ€ฆ** โ†’ choose a folder, for example `Documents\HDMapping Easy Forestry` - โ†’ **Extract**. Keep all files in that folder together: Easy Forestry needs the other files next to it. +3. Right-click the `.zip` file โ†’ **Extract Allโ€ฆ** โ†’ choose a folder, for example `Documents\Easy Forest HDMapping` + โ†’ **Extract**. Keep all files in that folder together: Easy Forest HDMapping needs the other files next to it. 4. Open the folder and double-click **`easy_gui.exe`**. 5. If a blue window says **"Windows protected your PC"**: click **More info**, then **Run anyway**. @@ -95,19 +95,19 @@ The window guides you through four steps, top to bottom. **1. Recording.** Click **Choose recordingโ€ฆ** and select the folder the scanner wrote, for example `continousScanning_0012`. It contains files named `lidar0000.laz`, `imu0000.csv` and so on. You can also drag the -folder onto the window. You can work straight from the scanner's USB stick: Easy Forestry never changes or adds +folder onto the window. You can work straight from the scanner's USB stick: Easy Forest HDMapping never changes or adds anything in the recording folder. Copying the folder to your computer first makes processing a little faster. -**2. Output folder.** Easy Forestry suggests `Documents/HDMapping Easy Forestry/`. To use a +**2. Output folder.** Easy Forest HDMapping suggests `Documents/Easy Forest HDMapping/`. To use a different place, click **Choose output folderโ€ฆ**. The folder must be new or empty. -**3. Check.** Easy Forestry checks the recording and the free disk space automatically. Each line is marked: +**3. Check.** Easy Forest HDMapping checks the recording and the free disk space automatically. Each line is marked: - **[OK]** (green): all good. - **[!]** (amber): worth knowing, but you can go on. - **[X]** (red): this must be fixed before you can start. See [Troubleshooting](#troubleshooting). -**4. Process.** Click **Start processing**. Easy Forestry shows which stage it is in, a progress bar and the time left. +**4. Process.** Click **Start processing**. Easy Forest HDMapping shows which stage it is in, a progress bar and the time left. You can keep using your computer, but it will be busy. **Cancel** stops processing and deletes the unfinished results. Closing the window also stops processing. @@ -143,12 +143,12 @@ Every point has these values: | intensity | How strongly the surface reflected the laser. Bright, smooth surfaces give high values. | | gps_time | When the point was measured, in seconds since 1 January 1970 (UTC). Despite the name, this is not GPS time. | | point_source_id | Which roughly 5-second piece of the walk the point belongs to. | -| user_data | Which lidar measured the point (always the same, since Easy Forestry handles one lidar). | +| user_data | Which lidar measured the point (always the same, since Easy Forest HDMapping handles one lidar). | | **range_m** | **Distance from the scanner to the point when it was measured, in metres.** Far points are less precise than near ones, so this is useful for keeping only points measured from close by, for example only points closer than 15 m for measuring stem diameters. | | **point_origin** | **Where the point comes from:** | | | **0** โ€” the main points, placed by HDMapping's motion tracking (lidar odometry). Most of the cloud. | -| | **1** โ€” close-range points. HDMapping's motion tracking ignores everything within 1 m of the scanner; Easy Forestry adds these points back afterwards (from the "Closest points kept" distance outwards), so the ground and stems right next to your path are not missing. | -| | **2** โ€” filled-in points. HDMapping's motion tracking works in tenth-of-a-second pieces and skips a few milliseconds between them; Easy Forestry adds the points measured in those short gaps. | +| | **1** โ€” close-range points. HDMapping's motion tracking ignores everything within 1 m of the scanner; Easy Forest HDMapping adds these points back afterwards (from the "Closest points kept" distance outwards), so the ground and stems right next to your path are not missing. | +| | **2** โ€” filled-in points. HDMapping's motion tracking works in tenth-of-a-second pieces and skips a few milliseconds between them; Easy Forest HDMapping adds the points measured in those short gaps. | Points 1 and 2 line up with the main points typically within a millimetre, and in 99 % of cases within about 1 cm. You can switch them off by filtering on `point_origin`, for example to see exactly what HDMapping alone would have @@ -194,7 +194,7 @@ The check (step 3) explains problems in plain words. Here is what to do about ea | "There are N point cloud files but M IMU files. The unmatched file(s) at the end will not be processed." | Just information. Usually the scanner stopped between two files; the last few seconds are left out. | | "Some file numbers do not match." | A file is probably missing in the middle of the recording (for example after an incomplete copy). Copy the recording from the scanner again. If that does not help, the result may have a gap at that point. | | "These point cloud files cannot be read (damaged, or not fully copied from the scanner)." | Copy the recording again. If the damaged files are the last ones, you can also delete them from your copy. | -| "This recording contains data from more than one lidar." / "โ€ฆdescribes more than one lidar." / "โ€ฆlists more than one lidar." | Easy Forestry handles single-lidar scanners only. Use the full HDMapping apps. | +| "This recording contains data from more than one lidar." / "โ€ฆdescribes more than one lidar." / "โ€ฆlists more than one lidar." | Easy Forest HDMapping handles single-lidar scanners only. Use the full HDMapping apps. | | "The calibration file โ€ฆ could not be read and will be ignored." | Just information, processing continues without it. If your scanner needs a calibration file, ask whoever set up the scanner for a correct one. | | "The calibration file โ€ฆ belongs to a different scanner." / "โ€ฆ does not name a valid IMU." | The calibration file in the folder does not match the scanner. Remove it from your copy of the recording, or replace it with the right one. | | "The IMU files could not be read." | The motion-sensor files (`imuโ€ฆ.csv`) are damaged or empty. Copy the recording again. | @@ -206,25 +206,25 @@ The check (step 3) explains problems in plain words. Here is what to do about ea | Message (shortened) | What to do | |---|---| | "Choose an output folder." | Click **Choose output folderโ€ฆ**. | -| "The output folder is inside the recording folder." | Choose a folder somewhere else. Easy Forestry never writes into the recording. | +| "The output folder is inside the recording folder." | Choose a folder somewhere else. Easy Forest HDMapping never writes into the recording. | | "The output folder โ€ฆ already contains files." | Choose a new or empty folder, or empty this one first. | | "Not enough free disk space โ€ฆ" | Free some space (the message says how much is needed), or choose an output folder on another disk with more space. | -| "Free disk space is tight." | You can start, but make sure nothing else fills the disk while Easy Forestry works. | +| "Free disk space is tight." | You can start, but make sure nothing else fills the disk while Easy Forest HDMapping works. | | "Could not determine the free disk space." | Usually harmless. Make sure the disk has plenty of space (see the rule of thumb above). | **Problems during processing** | Message | What to do | |---|---| -| "Easy Forestry could not find the lidar odometry program โ€ฆ" | Some files of the installation are missing. Install Easy Forestry again and keep all its files together. | +| "Easy Forest HDMapping could not find the lidar odometry program โ€ฆ" | Some files of the installation are missing. Install Easy Forest HDMapping again and keep all its files together. | | "Lidar odometry stopped unexpectedly." | Most often the computer ran out of memory. Close other programs and try again. If it happens again, see [Getting help](#getting-help). | | "Could not create the output folder โ€ฆ" / "Could not write โ€ฆ" | The disk may be full, or you may not be allowed to write there. Choose another output folder. | -**Mac: "โ€œHDMapping Easy Forestryโ€ is damaged and can't be opened."** This sometimes happens with downloaded apps that -are not signed. Open the **Terminal** app, paste the following line and press Enter, then open Easy Forestry again: +**Mac: "โ€œEasy Forest HDMappingโ€ is damaged and can't be opened."** This sometimes happens with downloaded apps that +are not signed. Open the **Terminal** app, paste the following line and press Enter, then open Easy Forest HDMapping again: ``` -xattr -dr com.apple.quarantine "/Applications/HDMapping Easy Forestry.app" +xattr -dr com.apple.quarantine "/Applications/Easy Forest HDMapping.app" ``` ## Getting help @@ -234,9 +234,9 @@ output folder. Describe what you did and which message you saw. ## Credits and license -Easy Forestry is built on [HDMapping](https://github.com/MapsHD/HDMapping) by Janusz Bฤ™dkowski and the MapsHD -contributors. All the processing (motion tracking and point placement) is HDMapping's; Easy Forestry adds the window, -the checks and the export. Easy Forestry and HDMapping are released under the [MIT License](../../LICENSE). +Easy Forest HDMapping is built on [HDMapping](https://github.com/MapsHD/HDMapping) by Janusz Bฤ™dkowski and the MapsHD +contributors. All the processing (motion tracking and point placement) is HDMapping's; Easy Forest HDMapping adds the window, +the checks and the export. Easy Forest HDMapping and HDMapping are released under the [MIT License](../../LICENSE). If you use the results in your work, please cite HDMapping: @@ -254,4 +254,4 @@ If you use the results in your work, please cite HDMapping: } ``` -How Easy Forestry works and how its results were checked is described in [docs/DESIGN.md](docs/DESIGN.md). +How Easy Forest HDMapping works and how its results were checked is described in [docs/DESIGN.md](docs/DESIGN.md). diff --git a/apps/easy_gui/docs/DESIGN.md b/apps/easy_gui/docs/DESIGN.md index 5a9bccf7..e3d65c4a 100644 --- a/apps/easy_gui/docs/DESIGN.md +++ b/apps/easy_gui/docs/DESIGN.md @@ -3,7 +3,7 @@ This document explains how Easy GUI works, why it is built the way it is, and how its output was validated. It is for collaborators and reviewers who know HDMapping. The user guide is [../README.md](../README.md). -Naming: the user-facing name is **HDMapping Easy Forestry** (window title, user guide, release packages). Internally, +Naming: the user-facing name is **Easy Forest HDMapping** (window title, user guide, release packages). Internally, the code keeps the name Easy GUI (`apps/easy_gui`, the `easy_gui` binary and targets), and so does this document. Contents: @@ -404,10 +404,10 @@ always reports the breakdown, warns below 1.5ร— the estimate and refuses below 1 yet. - **v2:** gated pose-graph loop closure (ยง1). - **Packaging (macOS, not done yet):** - - Bundle `easy_gui` and `lidar_odometry_step_1` in one `HDMapping Easy Forestry.app` (step 1 must stay next to + - Bundle `easy_gui` and `lidar_odometry_step_1` in one `Easy Forest HDMapping.app` (step 1 must stay next to `easy_gui`), with their dylibs, and rewrite the library paths. - Then re-sign the whole bundle ad hoc, because rewriting library paths invalidates the signatures and Apple silicon - refuses to run unsigned code: `codesign --force --deep -s - "HDMapping Easy Forestry.app"`. This must include the + refuses to run unsigned code: `codesign --force --deep -s - "Easy Forest HDMapping.app"`. This must include the bundled step 1 binary and every bundled dylib. - Every bundled dylib must be built for macOS 13 too, or the app's real minimum is higher than the README says. A build against Homebrew links `libproj` (into both programs) and `libhwloc` (through TBB), both built for the build diff --git a/apps/easy_gui/docs/images/easy_gui_window.png b/apps/easy_gui/docs/images/easy_gui_window.png index 95fde85c8d31b5c87166df8b10319f1f45927624..da232f4b878b990e97e17c5c2908e8c24c69a4c7 100644 GIT binary patch literal 17165 zcmc(Gdt8!t|2NfEZJE{D-G=3%!`3z{OJ|-!En6o|mzF7>Y90d*Gg1@;Y_?p>LnkBi zlxt+9K&E&GI&7k;U>-nGVa*a1kqm(WBF`nH*stf=Yxn)!uh;MTqZeIX*Z2GReva?Y z;kxjx=lvQx}_2C z&95ZX*s`VT_joPL*zkPsgUlNFFN-d>{>#E7bMec^-}$AsZ}-tp}@8{vtw*0ju z|NV|Jzr#NqxOUEdy#E|};JuI~8{B;}ez{~n^66mD2Nw%}99fk-k*6Tt^C@8wRVlvB z3CE9jE04+Kh(bked}5;TD}WT>dieRSUq8_QJmC}@tj)f75kXp@zhRkxLOES9+N1y9 zPn1vJ*ZCZpMTC zHj?of_lw1+@1E!P$hQovAh7cGSuayirSxh0ql4c^GqT%Kw{*w3XRgkz)9tYmJj;|J zw#B-<4|aZin^Mh-^RZdru2GbW9gr?vqf9q&)JVh7b8qJI7;eX486Kf4~ zpZ9p&F9#oRgWn>QXWZur^Pqz_P8(y!l#TKy-U@$IAa^{ymRuo|OTzg&DqO>6rM^|Kb_cg@V^Q~YZ3_z+Ox!v+ z(4arv6Hl+W&2t1#{Oh)!I)gmP1FenlDzWzyqt*c3-wcFXFI9gwxOA|Vk$mER?)k1> zy?f^$Y1&~s+eg`|t1il`o0^NJM~YxZKWBb2crO!Mx3kjxkgO>FR?uXgynT9*BQ6)& z`PmKN4y;jVy+W3ujl@oR#GE&sw zu~4*3)`x((lu@;bf{KjJm|fYZ5eczq(X0MRY$RrhK@H|?XO8|R6Zpy@SMddm)@2Fo zV`I`O`^ow~CdcAabb*gn-7h$dyIa&)6=}(AOTZCeXE-KYcdpSeY0XHFCXtp=I4JwIr zr!hbV_2~U>@yQRvEh7H#EGlMZ4%8k;@Arm-H~91+Mh1uIj8LRE+twV~cNsN?p%xyo zuhbsO#Md$t#0f-sol_zf4@IU|AxYCBv68QnMv)V~BPI~L(7q#azuZtyL&1?gLLZ8# zrAOokqnV)tMUpf)yUcG?Qy~)`Atum$_YFZ^(ye3;WEq3eiYX!#QT)_v5>?t6byC6uwWe{npGk_6LPixBE_!#LhC6DfydeP@tc>H6b|23> zP%?6B0rK z)Tbe%ZPO{57M5M%rs42v#xVmi1W1Wo4dvp?#Sgkr1F960J{*BV@U{8&dR-wklj6%I z1oJnewCDp82??GYyG=G_f@)y&%~J{STOPyXq5Yy(#YYT5-6GB>yUZ(XQe|%S3tiD~ zd5$B1qqcYnTeWayE*Ml+wexGxO50h>YqH2N{jS{2Xwb;7C8e62DtliO83^*TdrJ+) z^}67w^HMX>8o0{3Mn=S&W)0Ya{@us=`~SZ|Z_aS%7JB#2e;7~yW}&&wfTf2YzI++J z&C2ScW}WUx26xLJXgzPky7J=1i(i{Xyb`KNoGD+VduA>a{$%vH%n`A@Afk?XQ>l-g z25Y{&`^I9JvHZ$Gzpvb0f7^JHG-xb)(j#3SgC`1e)k(UQ48E*m&*J9*kNvG)|M}-G zME7LNA)8Iv8VNOfts0%J!KABjQH%c!6nl5{OIPS#jbdN>D|a?>0g>PV8T!MiEz;8h z$+0~B6Be-9xN~`TO@tcO7enK0(7iiP$%7X^|NZymD(}_b6OCvhX%;Pu$2`JbGi0uN z&|r>T7OiJJYTUGZ)SZ61$KwH_Woy4?7yZ= z+xy!HR)4&HZGG_c>C+v@o&<#`$Q_DubRm(!;L7=<_MQBPzT9*oo8c{L=Wr`JB zuy)}?r;Tqg-I(rHqI1cqLQ*o()X&X@f@#=-!3}MkVw3Nn;O!%u*Lv&Ux7f+<(?gR_FCj8;1V}zZ| zp7@SFM1?F4_62QQH;Mt5)%w>rP88qtt7k-xBI@Kd+cXUY^+DHF4G*qkqiAcMmveno zm}>c7V=pmCqFVG4v7f~7+9JaSq5D08ka!i^kEGmyW(C- z{9#6m=3MvQ#V3Cri!P%L#*cY+5)zG93=ogPn4?{$HiX8*ltBRA-9k{^t)wT39+lqQ zUR6AeY(+rsVNvOos$H;>LxH*A0F2qzoMf&FQ6W$H!qyCp@wAdP5DPB&!WxFt zx4b^Qf#xhgqXi^Q;E0}5kDwJCRetbSHnBt{w?&v=tKDYH1*qj}yrsp}jL7DX-<$It zO&pu0KD8XP@IJ(F){enwe#S9zZL4zVUJe&}*iz&!;EvSx3Tyb)^vb$L)GqtdoG(2B zzzp|HQL!{Gn&jlfwE5EQRwC&)zV&W2Bd!S1n*Nc5TPV3-kg(9;Rvu&;?QsgJEJ^ey zN9?f@*G?*jcnin^QELfR{6{rQ;h;6nY~jN^s8<|%T2!?;$z3W{J@sZ1$vJy6O&D`g zB+q zbf{Y6B>_)-%0*=1iy4w0VST}$F_UQt&FH2z@T^l_C&m*V3rCXsen%P4nfoPD&K?2I@k(+Q74U_-o+YrkxzvdNh*6+z)4&CRp=KS0n)~&F`UFU4bbM- zb+eC?Qn14>5uR|#5&)8Qsdf$Y2@VIuePY0@viU)ArL8+i`Z0*T5hPj+y6^dxYr6|1 zQJ2JB8{V_9P6e#&msu<_W83~HAyOXp`GS$pudW2`!hwqH7J%jy0^n?FPc4v$rn2)u z29td^j#&{t2oVc1wu* z1#|M_pbWRcoa?jc=ZOa0u5+8NO|V%O5k@Oux=bFS?WxWP@wUHhdC=3-bDyb;yoOp` zBIj{55|P49;Uupqmm)`%Gw&N%UU_MEQh`H_Rp)3sZb@qEr3((Wp;zfDi~whj?k~^1 zMhw2za&3!aKm?!Nq>hf0DRo)p29Q?Iw;Kvde5@`aR5VC9qupv7jH1+5JH=+VZMp`Y2}K4k>F=}rart@9 zmURK5@5zH5euo>270;mP>(eymG~I0wp$pM_Zr(e%lUmi&ZgjgC>=8<-NEBC=GvY$f z9;Acz#~D@$ftG{eBhYKqp^-qYF7ck@*sn~Z$7ilQx363iwC3p)4;h5$9xe^Cj2qLG z$5&Sq$77?i^*!zpc}BY|GQ&}xq{DuW2>1vT zeXVUJIj4CuVkU<-c!!xhIyRGz$&F%0^bJMAIEn`@YJ(b}BKNR9?>9nn0sf zM;eDgmy`a598)G{Q!u!IA{>paz_>8fl2EwM)9o{*5QErv=+VQKix;Iw7=J*#FncZc zsK`5d>W+^)lMw6s?)`4Jv-8_KrZe=6A>S&`m;ki}e|m%t{m%!z=c4a({vcDk&E9*! zgQi{Hp#vCmTn9YQ%G@7ZyVmGRGIQ!QA0^K01p@tKa&GmEb&>P{h2`N|8hN)Sbk@?Iw%XeKwDvZC{8Won`U_j07xlAHJI)g7oZ|$r~m+6aq2I@@Co%-D7t4wZKU)!c# zXE1=ak1CA|ki%@NgaUU(Um(}63K9^vU$?P=wIgp}uyVry=dKNw3<)8XXEbNDqKe~Cfkoq(w zXh@4Yih_Kh;fD_Jh7flYSi9Iv#Na~}m4me-?+C6ekv~y`=b}BJMPJ^(3bAadtfyzY zDd77T^}2rkb37^NVMCB2AqyipX*%fBX!O_6r=mYD#4q2-Z0Sj^&k>#>sih7}q)QEO zB)53+b9xi4z~WM@b!eMjfenAPUfOwJdF8s?XGveU!N!g56)XJfs|I_dnPP@-V@vtq zNyT3s!j?wZr=A*Gx+-)urN~hWli^6p_ECHPdxG=MR)<_dV3bHiNF-6|EKBykrxlJ% zI7RzI;cLh-fp{}3Dt{xhDwKqc6Vn=~inBZkiN?UMFx^aWDY{-*?cSCL1>k0KAB=$? zWqOOII*JfOs;TNpbW<1Gl)z<|+rAHnY=&ykecmE>v8+>_tFVzozGY*77*m4bgsnNJ zyziM0@m9GYNiE(^A7pE2?veMZ9LesQ>00iO-i#7oEvIk=t?O>ky&zclFI*@a79#9Y zI#adKAUu=vbI@0v<#P32(O_M*q6y=a%6)C-kVsd&ZXn6{P<2hMgwmCA1Km(LJ?)|Z ze(+-n0(Xj(M3azZnz}O)&-X+1?o{97lwoR+y!*Rng^l3Gi^3b&kWNjvYF)bW2jL;& zM6jyvdSQq&($9QHa6=K;%djkz4J6E_f$52RGD-dkW1bIu6H}Ix2K-RMM?Hz)KHRBj zW{_AO9V2WSPcW;0LXfG~Igv|P44&9}au=yB{p-3#sAvWoBj5}?ke(cq#fAr>i@rb< z_%JUjnw*8dbC^`crG#K%hs2+jLWc%0s^x*bDxo5n?+)k@r_Omx_X@pS`dDi4TzLs3+?!uK_K?2`4$I$%vu84#n@%H;8n7b#u!+8^=$p{ABDcIc*Lkx65Gf8 z+0UTW4Rfg00LSL=@4JQIx#0be;%8cH753~5q`k5!8#jC#b+Nn7Ojv&?TRkB4&}fDt zLjW&#b<^g}n@!x%CRU$Xr6xQ=-7(!BJP&*VrJ%{e9p8lgursBA%0^-eG1BawiM38? z4zUemp(#D(E@2JP3H?#{osF^{p`bnCdxcO@An7R*{V3cmkRH$`Yj*_873y+!X1Vz5 zGt-_B^S98}ozsoiZbpEPtlo>>iNmE^pUAM2J~Ea!wf9D8{g*%l6NlH9)e96r4~4m0 zbN;Nia1rRzY&!?2L=j*l(-~Y`abJgvp|%$8;aD>?pD6YdIl5D|U@@{wz0Q8 zaeYvC7#M0f`0et$= z7nS^o#$TYn&x6?`w8hW;+-x2t>HPhrgM`V1>xGd= zoNf~ZAuyq$9PCQqog9%zQ(XcmFRxqxgP6hGE>S)T=Vo&d#!fE>e@p;MF5n(SBXqr%%NfiRPbv1&W%ZAg`-px#^GUr`hBib zr~mo&^F)GbvGCXo&I*O!$s|oI-n!O_tY}-Cn>KQ<(L3U4&!=_LTXuOp0*nu&=V)JiZoOc35htUIWf#WariG zi{N*fGrn6UJzI8KINZbYaR<(D&mpX#J#_Iwk@O^tRLqC3CWi-#(svFmD#Tq2!C$|Z z979)V)>RhHD`$s^!RP92Rr2sPrg`yrPxxwM8ih(-(&|8lvu&g{-@q*iuZ??%+=FYK zZlFgP9121#eUuUx8W=|~PXsdh#_afE<$g6gy;jwo^vHgpA@6Fs312xL(baaWB6=*j z(X3h`6gq~$#n#+pbS;b8Rzwu{qMRCyJEm$1I^EzTBT_9OxbMaiSObLwUig^u&ba+H zOaV2!r4_uCYW?H+!fJWSr+{h>)=k!3+h?OIG_mcf%&!-EAt-4G*^@wSFG|hDPIqIo zWU4@zh89T{K4};FN!Yi8hk&jOVi?ZHIa_9t2RS#0>WEUB&(=pZQQhL zoPIr5-%l+(t`y-U?4fL|#I07=qdqN~*nb);*>vAJR`HJQlcm@y@?5*CG^gI4Ilu&eJHiw3!mi*mJ6BT#1cqsA^*kFMt9))%S&{095#39eg@ zNI{o!-d{?AD3?jttFswe0X=b&jXH9^FpfM3 z>(0;Tk(nG-X*byo4rAE{51Ezd=yO$lxbpMn%(cxF?5pOf*H^Q(^{&s;rZg9=M7H_^ zG(BB-j>}iF(xrV2*LVOwq5H`{XHJc+U(M9 z93ni`v+bv~sD2rHMGOO~0gO+#T2lD2;&atm97c0BTG ztKH3#xHJt2>G!JjYg05p!l=YfuZm%coZptGOnNdz5V=|{fu2NLoF)ASy#D(nR1~S9%7&R5!E@tut-eR6A<5V44gdikU0*&>TvWR%^~8A@L={9|+V435(sJE(BTlF{y}LE9z7?;gPYkV~I_jPWg>9U5 zFDD%AY!_V0g)blXM(d|L<~#rZN58ZNqbdxaT@jj9_#xj==S>xvS_QFRA6-G}oTCerv zbM2})Cr3;!AR>%e5NEwk!SX(GxjGw50OUsM-qep0*JG&f3CYU4Ll!y$-h$#sDM2?p zt=H&ZHvsULHcNqGN~e1!tvvTM74|YnJMAVcO@M^55yLe%Z7tBB7U>UTD1bAH?YNo0sYH0vW4_fj+SDLLi|7SA)_ayxH zGS^dP5cRt%JV&DuvSQL|O3YBDRcjPlb%@r>$By5U5IX?0h;@D7CbkTvwNV){UZ%8o~2w#z3}m-FoNhIJ=5Cfuwf2)=k)}K-6n- zA^T27uv{;AT;9OpVY!Ngs3}*+{$A&=-D>2H+Y^}6?Srj~B+r|;t2Rh9xF}Sp?n-#P zYZMU#MKAUzDd}CPD!H)8Xg`pe&GvvT`r0g7n_3@Xaxrc;A2H4FKPIN~l0OLh%xPNa z$Wkp80Wg!F-eC18(fXDhRzAq{RzaPkJrlIaDL7$GYq2s|RYMT6yAmSC6N+hS$)={5Et*%D2I`I7 zHmfSidFylyaQE;V%iGS~C=IIG8H^&Xx2@7b3vj0@ufrn9Cw(n_Vv#{$l^Px%wbhp0 zSQs}7Ao_|)=@*$?>)@8DfI3KeR(7zpbxA}%=ca_pApo5Kg<9C<;g)-}FEHdfN?WK# z{1Cr<9y7w~!RNTS03WtKI?8hWRE-u-W|(YS4ZGjoI29|yR?%-a;Gw6f6vVK!sjjPW z%4g%Mp}hShQlvW@2|k6PeUjw4XkI?k8w-@#+a7V?tHw(S_o*2c8`2KOtm@g@7-dDZ zCAniV;JH8Er^{MX?3^Jb3+9!=%be0)Id;Z{3<=sZ9=b8CGPz5^b<%Qq{C1%ypC~<> zz=|8(T`zl;9}Gu((J0>H?{+;&XZ_CBju zOPHM4yO=f@RV}Db;v@_7J8pzjzQaHYJJ)oPYRvGjA8C9tT^?Ij0BqlB_1p zZ*1l*X?3c!1hjC!JrKL*eCpoikFT{fwfqMCfl8hk8;b#navOoCfb#nrV-8z6-tE5J zg}S);TtRpJSS1JdaHqcuNCX(3L8HSi+Adw6=zClHaB9K*ZGdp>bksByPNl4J7uHbtuUa zX{IeXeW)H7z8Qyv1&O0*?O8n1wBvIcbLxqGv`pep7-vpRD%f)YWJB8bHRa$GF8D0d zjG;!Ttv|Hs%woIkz5iwT>hRHcxM?(Fq*+UzX4q)%XDRIzD9qSjUtk2x zi2Li*Q#}fVTVHV(=B6YjJR0RyHU6L=fyk)=tMYrF?y`=nCUG3g^Tn;6TUKHqtk=2?Hg>y%ZgC>@gKCyQ9fQ#%B^w1b`Q+Xze(c*BFcf+?M+w*KU%W#lL8YBUsb!5^YkYQB>Otb3!X5r6%#@#TkXaam$UKIuDwX_`i+DlT-0I=v+jTGtfdK!7PdDg$1raW20uq|BZiaIziyGOv5*Ix?sE-nnK} z7`GCO3&pX!q+I?pe{^8>z%y&A050w)+d2n8LM*8qm^l9{8jD2b^zGp8nBfFZ3m8AhA->*z&O$hz(0Mjb3mACx&PPTFS_> zU}u@K`Y1!)P(CK0Xo7CH$q+HPPv(Nf<=qFW3LK7#Qrt0{t4y~)gmRQ5&R2`111XAN zE=Ntpatq;5c9nJZ6y}$b$J5_NccI0d+5U6roOn&jiYBZ-( zT~G}dG)I(}HEyOU>I;}dh_S%01rV3AG-t}Ng2n4pUz~XGyMpKjUN%$dXra23*w1~6 za0o?x{O|dLxA7C*qf;5PGP9sXuEc5vvp>Il@Ww{u^_(o%>O?xf4Wefep z2}rTTYUtU)XRYbpqTYU|2k2znL~^*PXsSIS$5GVljkAJ@C12271n%Z`a)+IAdkG|F zmjJC)%Y3KFM@oS*ohB@%h#CH7KI`H6etV~SM9!v9J`wu6Exu*6_a{jW0I9kP7d>=b z_7L~||7dc4iEl>6kTd85(iwl~Usl!9PrV;jIln^tAaLaC*_h3o;jPer31jPN9BwgghKz3fv-sv$1RL`J28$>J^KkXxJLx=OyTqjm!_2DS zz?O;CN^faYv=ax$_T~oE1zo?HPEw#nqLvQLad@Dj1Lp%@XQeUo9(7 zT->-7P?4ZtI$(DN*xFFD2ZpXHs78wLpML%Jj_(LCe&m=we#^WM=Z2G%yCBCVNZi(S zs3593jwoNJLc^IKkWD(Fr3nwf2(9CL{=36+X_bDm{(>t{5 zb;=3W#CIB?R%1x8nk+WuSaYGIv%mPd-{Ul%lA!`fy_Ma(yFhQ9KjdaL*omUG$p~^y zCdN5~#-b`h#6prX_MJrzcTK?)m=8?H`|f4tp3My*<|l)}#KUX_RZ%2^YGa;H4`H37 zV=$*=$#8mhfQxK3aD+Km07PxDUqES&X%j6_SX}TIEilrA^4x|i19(rRR43MF6i9#> z2)wer<*)p@;1x7BQS2?M;WtK-)Rx`OZa6QoS`<#Z+g;fEm)-gdQnIBDkck?)v_oeG z$HrL4t^@|=!fyNabdScVLu4P#mc53F^O?ygUrFfK#T?q{9M ze`GB^iC69*nFhqTh(h|r1l)a=p;>bMsYUB=Ms!ie!_aJjrWOPKG1uV-vsJPUm22T9 z04zUE(o7H6o1c$3Z)MAeOIjg`B?HBY#DoGjw#{S>&uImoH`0S*1W+cNmp0}at(Cmb z-^>d|*V zX)5PB$mp;$GkeAIEHHK$%W@$ru2F5{|N;qzi)4+qX$2NaQWQ{9A$ zc5H~+vyrc&bxEU!(Jkc?8dZP>`l#6uL5EX5T=HoVZ-o#1PZnk>uxKqx)8MbFDHr*V zC|(CQ`im7MJjrGL5+TO?M*W>oxTv>SfhSm}9HMbe`P`%e8Vl;HQtU``f_DEx74$mPZiF^wryOGXT zue)>vybXS8vm~vsb2=E9G`x>zi|4BwC`n4!D=ef zq)U~h=D>c9-kSE~8ER--*2e38aeg~}Tp8%>$-uJ-7`kbI4O&#Rs4`u#Tht;S`kgaQ z=~fJo!Wv~)9BPi@lAM1+&Ls(e*KXcujy*#84e40Yjx04-n>JDV275$bvnJDI7fFhW zguRUF8dfw-kcqsN!(I`}4Q0o%O(pL)KI5npEC;LbjL5CB!=be53SpWI)oMo{PM?)J z;C`Z?a9yUOKFU4UThaMPkEGue-5$=Z6i1@nY3oXs84znA4t+fQFx9^QC5sL*f||<5<-gLSIXCwVhT=uEiLU9 zQiW+rOXkEKkT~ZhBbS3ytyf+!S$1OA0?zvx`^EdxHMc@`Yy*i_s+QmAduxs#w(?Hk z=K^Z2wJMPfz=JhJ8ZeKxcd?0Txgpo`E!gc>FE4_ofs~g`zw5pF<$EVEAU1MxQ6o5& zy-c;-VvY#l$8VnI8EZkWraGLv#+GPe@)!J@F804x%6WI8cc&(ONA7%{)H3UtFNk^~ zM@0HDo%uo5V}EWMsTcoc&!IaTXX4IVvG1hgU56gn=O>A3himaQl4-O9PziLWn(AtC zh9I^f?Nt&mBpOL;0tSv+b57xNBKKu!6#OhWks&P9*MmWC&ofozwAOJbc8lOI#cByv zf`)Bv^L;()wE=nej~;f@{n0?e1A=ba zX|k|9@r&CYjsEw#Qh1Vd2+g0_0f7gO$IYvFiYh-G4p?^$d`dMOcj)|faV`Z*BJP(L4mnr z&6FH{N8m+Mesped|Us46P){)v>t!i_OD@^ z&UkXXEjLNmcAPuo3@y|v7rtqZ-vLw#OwthMI$8~x37}BDep_qZs9Je=l&K?DQ4`8O zu6A~hyw&Zv^!H5I%4>Xu`u^yg{@p*fHZl@PzVhPFKU2Ei43q2?JKIid-^Llr zsrQ2G=s0ogt(lkq>$jI(?}5JbhJTOerRh|BZU-O`wcF)>r_vYw4JP#Na2|k_`(5uhSIwG3sx#`T_YuvpW~!MU(|$|+4cE=Vm{%8OV_&z+X77Oh1~xZ+ zzVO@%$Qid%z29)%`H@A#OmkNC+ij>m=L4VaqH_mdOC_4#sb><*7eEU+Bo;D#fCV)1DimffMn}Ke8ZIBZ zX?Zn*=uuECte;l8i9ZH^yJfpGx}L%jJ{wQCxnH*su+?*7XGrUkScuB^)~j;G%c<$K zUD_2^BQqsO;L`8(>WQQ&PNP~PwKT8E2`EBa9{3B7)4ZIL!YL)x*3WXk)<3*A^?pW? zs=261qGiRs0BCPGR`hCmnX>;AtJi7f)1`$ogWW(91audZ#LXc|xWc39Af=c7StCXbw}7E>VvS`6SsEb+)eEgQ;Z?8WKlq! zJc^k7wL(MuM(2c9E_iGb+kXHnOY<7S7o|)I-v7}2g(M`!pYlfjx?VO5>6|Z^ZYz21 z?&W+vAe#n|NO9KMDe+uE)TrZ%aL&5CWSg&TBFB{S@P_VKJSQWD?FJ?=r_piDwr)u% zQy3`j5UDy^G`dy4<+?$FC!a`?(hKzpky^Gr?G=?P(Dh971-MIIxaZj5>Bj1tbUqL4 zK#BxqlG?a=Zn-zzLzq1i)>Yc74vCI00R5-QdXON1#kNRB0!jbVTk=eKr33aRgREa!h0M(U(d~EopkO>Q9%~eWHU;OyZ%ut)|M!BF%l!wmGgv3T`fiGzZ>h+ls z{ntm4;C9`1t1QZ10_u#DPcWJy3o+XSKo4B@X1}AS=fELDyrMyL!RZ^XzxYLHBz9=P zrQqTcD;TEZ&|zn=cXWK<4ZRx^O^&8{H2|aIg?!Hku8W_)ETz0$tDQ83=w%1$eRw=W z-%|F!mzZbt#ZN|uX2?j*L?4)`4gePrEB5%_IJi`~Blk4i)Gk!QZqVt$eEhK9+US{f zTo=>kMS|I{Tjrct(=WD5LolyCm>pG_1DtbD%th6|b8~;=wEo78{d-HNX#1H=m@CteU literal 17213 zcmdUXd03L^_cvvWGg;O+-%87+)tPCrG|Lq#n>L#)oh)&yaVv1i2u%=h#>)EUQkj~j zl9d)Ik}0Ah(yHc$8G?aGWob%4hCqP8`zVRd^u6Xc@B8~*?|Z$^A6<_R&%K@d+~<7G z=bWeW`}Xdb{jJ%zAP{KwuASRFL7jA@=}^?r!O3U^S!HwuHI5n{%Kgx|_LEzPxQU^|bEE=4pny?h*{=PSrh{XEjZCqwnbJ zC;zbPxJ6l6S=F_*%r6tsH*;ue{l4Y6`{nfX^erUkEB~UXuv8MCLJtaLU+78G?$d_9 zt0lYVAxC!v1Ufgq$m@q^90_f@n7+OuQh9)?o8_e~UzQupGd=BiaTl3X^r|lBY!Ym- zNTn*XpWWXR8NTLehYD_A9&uEg;!1ZjEg^+5M4oMsVyAX{XYd6<)d@F+I%7O?` zbj5QVtiAC3o*nA3-$D8XZkzEsGW2xbfq7AUn-z-|ypJ7w@95)!0pW7nFY022{q1|*^z_QrPQI3tQ2R>++25B$F7h9y z>SSs)-UJY?-|aF}H(H3HWVUnjhZ1K?r>LS+)Z50yzK{=cCOMgi2v9x^BGT-W+MI?^FBymQB&9zW% z3lMvV9)4esUL>U3Ec;nI&tt@i(Z-6)p2k`*Na<%{55cvVg1F@w!agUu zr@a$AMNkNzG3&7zCbQ=Hbb(w}hgXO1mU5*PF+WYBS7i5NQIAz*C5f{V=kYESeJ`hN{_I zDNQPP?k9NSDexqyN;>G<*2@)S+qR;%+X3vXvW7Ay$~A_{63=Ak`;;-&VFLC|PeD8} zxrL0K6F7+Q6Y%LRnEXQ0wZgvV1+qhf*x2x*BLaNX6wvsg0#28Y=$nr$vgabJ4E()f zf3NGZ^xFK**TVefV5XK)%+3&7VtY~71R@Smiu#or`a?u=d7+Xfm4?8m!=DrdS#j;k znztGgxx2bYTw`;Ts)edoSW3)6dc|!-!RE?5o!88AEn_yDc;>(TxAm%7E{dwoXeQU` zX#7oWU-9{n5h6NwKz?8NTU|KZ|Ah7%aO_bJ!F~&T z%lT$uSb`$pjhg6u81qoN#o6Exq+%7QqJMZe<~#l7*aOzYvvmFv(537t4`6UN1JHiG zIWT&3YU}F!xsM$TShjrPs>7HqTGIs})-Q6KNefp+=H~ILcVGVh%Ps!b9#?P7^gbAP z;&ePNF1xw8`8SfmnCYFcywqzikJtNL&dtr;5a*|=h^eJL*EyF7r*vt{)dOL3ieBuY zc-vX(CZKn#fQf#wx%FXy?KkcrJ1v?VO31td>_RHNKxm7EAfk-NMw?JoXrq;jE6r7~ zCBQE?tTYGfbL{NfGHn2wS|&z+P?-;E3rx=;$nLuk49Hl^>#kmJ!p6M-FxImg*f&LG zEzKBl@kYzBH#RfJ{qEAllJLTnuRKjrY?V2BF`05R ze@%;SkO?<)XUkQC?BhwA9#m0x-JVB0x~1>6d$O3qHfomUFAG2XqfTcBuWs46ulc*N z5xYuXyx{H{cjlW%p8TMjh#os#hi|f<5?l#Qq%tjZNJ(!B0ExcO+OuV>!Ni3A23hI% zS-+S1VZMIyzX#g%T`v^%b<7-9o#l>>jstgJm!_c)4oJFq45nO6rcs>QDmLBglpYzB zl9)~vQfbLB#;`frj{WW`-Jox%eW;_^>4?QE6g&Vnlhw|ETiUqe@d`t|F96iSbU1i$ z2@QOHziQWIfjT*&|I{bg0gvRSw;+`N%p?mjQ-GH~#~Ah{3Lfvu811hNol9)FP-c3q zk1SZjrR{CD;>w~+G3r>1!GummgwC~SI{E3X4&qF5J{ns?JFnT5ixu1#L6mE#csKaN zPNy~kc}bE*e`L{7p;=r|8l^M3lC-g@AZ9Jk!iyUsiry&W`)1?Z(-<8Mo8-}Lw7NY+ zwVl-Ab}9oZIBY|9tWQnvDZNJ2db$lD2`sgLF}syxpK9FsgoD8o*aMv2nqg^-D8^?{ z(gal&NTH`2EIbBOJWg+UmWo%{M#^HClk4z@1~oOZs_YukS}EobQNdRB=N8hlev&$d z!Bki77W%^SDW5QzW{>}LESGw0WMSV#e@HKKcXf0b-c{)7QL-2FTM@Hr0GYeyfXT_y zn1K-R>WuT=gATs@Qns{kIQ=4mXhRH0gFbt5RZ!!2FtpyAw8K55hm7TwxLWQ`X$T;_ zaQBabh1~!guX4$Sk;^+tY8$!C+M|yZ19Y_7}C5kx!u*=NhW=5w%7bdJsvR|9#X=%My2yTje3d9`sZM$pq}d z;I3sMCe7J3@nd`XLSpAn1Y~XlxS#`$W`TRi$>rX%O_i=_Q+85n)CWaI@P+o(E1B%kr6r*$jU zk)cghCDQ630p5$ERNT#A7}Mh0jtMn~392k8{x(|GQz4C+qPIea=!?R_VoNW-ee=c- z4%%NnZJo!ID-H!No%SNLzl(aw}f& zcT*Dt#;+tRL6g$_7098xtf3DLiEwvj>~}i^a}`p8740n9aX=Phi^uPK1?p5NIoZl z?;d=gIpqFr*vS>ofa{L1A(A4KF$1_xT8tX|KE5GxQ;cRWZ_IJ&t>DG^wuU6Tneq^* zxiS{iXq0``%WmU9F<-7$bqP~je_eyj9UX5%BcV-JE*tHD5c73=P82bG6I=0%Fo~y@ zb>ugMUe9!*7uehPn^_lDrnTH)-r41Sw~xnEr@E4=xgn)S#18vKzLqwHswl!x2fCHU zi$;g9ENIz0I)Yva8gpQCQ|&84Uw1*15D{7RFxxqZJvUVZb(h zodMiXcKPK>1LK`R+4){@lgzemXbsCE#O^dasAkzbs zL|F_XFVL)Y=ZbXO)>NYX#u#d0Dn%}9`F&4u0r^cTk1vvu8BqcWNv^09OQ|>jMR=ZF zGTO6V2FcN5RLx~SzuYubqTEF~k7zhRtv@UXs8&_9oe3|V&+ppfmK`(N=pO%1;q&NG z3s&Tu8U1N8`k8zGApk5`-sriFF!=24^;V-VOcvbc;BTky_Z^FwCw%2sYz}ag_YODl zZj+aewx8U?|78Y92e|(31dq%F;MC%nxHv^64&vyTjf8mdwMzAFOY5L{{kyRrCt?&ponFMJJTz(v?k4W1 zh_wx&J1u-~4(&^+Eo<1Xba3y)E{4v!A46;|v!4Z^iMnVu*JTWAPY^>(6F)lKzD)^p zZ3MvR>Wngqy&LxkAtx0fXP0GXyR&OtVsM3N<)JAp)$B1F1A71b%IEBH2J4*Bqt0|k z!(R9Ti@TS;8Jpg=+2f>~t>H4n+z$8FU1QHDbU%Dm;>R~{T;PnwB^hz2)Ebq{+Tb!V zgd@9G_m|c0Ph8g;yuENJC!Csn8t^@Q!EKSa9h+2#CG`5$@cW$V%7iab&N`+~ug800 z$MpC{+k%10?dCd}CFSM!?A2S7H`d$#>z&;2u4v`q1ASs05oiw7I)&rjs5eF;)HYh46bUm`v;`ghcRhL_%(KRkBV1i z-ZfLSSmyocx9Cbth_*^C-U6d)ZD9JYrqiga%zUh0PUR=th4xt>c%_lDJVCdXi1q zHsgpH;rGmCc4o#QJ6rR%*blqF*)fZ$^fEQO>6BxP%ZJE76z=7r*^dcg^c_f1o17pa zd287@#>CY0LS@8z>5XA&p$xTA9-WyZ_9A5!DsduBq#CuMGW0d49yu`NDFB}ccV-RB zS)yiNs^uLFpQFMk+M_*{l4ApLRAw!~-ZC?XLX@Ei*wy4V8?|FtDFf%uPuipI-j7l* z_Vv2QLf+^Ub2FiVFcMzG=*Cw=67|gK(!_Ps@gE1t-W<>Dm4(V9rPgoyOOZZ;xS zY#MKe2tD=G8u_K5ezXmn3zZYelxwPy-K|O04ekQD7QslW?iYG1X~Rx@j#>m2lib7R zx4Bvt^#rnO`2K^4!EQmbr*J^iqRv*?O&CsYqg8Hk=BuupZ38aL;KY8wPrggRJPI!5 zchrx-14~W0c%^|;-8ZQC!2dY$QGyPpA_U1e5iBzVF5lpes6FJJA=SCnXR6F zmdsBo!KjC;4dh`W#>9?$+9V$5M@d=AUp|r9F}_>{QT+_64%TEF3kU?eVHYBswIz80 zd%e2ng(4fb8P5I}o{te7H-D_oj-IVI{ueje86w_BJ-EHEZ`Xjju$-@vFW+@H*de4K z`6o486LADQ%vq~~l^ylyHR^qe+(Rb~jKDCdkK-RA3z?^EFn;a~8@zK#+Cm!!7}LN{s(YWxqT?2P9F0GX5e zD>pId0l+;weB?Y2=5tDo4Q7eW>d$T3=Q47I>?uuwL00^?ki z0;KyjAcs+Y{AlYV@5*a}5m;x!>I_BP2kh<2AY$+8P7PH{A0C=*&PJq^7YMi3+`VL5 z5%aXbEF=hyxYE{gRj{jvAU+8E%hNdix+=1i^OGC*UAroc7`;g*4T3i?d!pvI-Quuq zR7&|i92?d^5C)>vf{@xP0fBQmYd@?=ypbUgdiD;YjElY&-Ca4vx{zgN>=Y;mBx*6vvbH302qw&J0Z`R zsw~uZs;=ZzPvO0j=t>Tr$o78r#QIa3Ybl|@Q95!qG=`aP>;~_?Rc022kn)+rvy}or z2`OXGgN}Z-d%dPR+oQ0Qf{?orRF}L1!*?F{3ryO>Ol0*@>346^@G$GNlJ|8czw6B*In6(AjnfPljIvP&wAaLY< z44{g-a*o;F?0OV$RUCgA5iZOxv|F;f6J#53$gS@gtN(Ea`Rt)eEsFF@RVMPdP<_}h z+js@cgjM?E2PIqX1amc*h-G#PjUMsLFe=*%sjXG+x5?im<+C{)gTBRVa(}iW`v#fr zsW{IhH0)aB@x8nQN3;x!k{ppc$U_{iva7(u)H#DQO zR~-#LXlhIDynU`-fFJ6M8tA?16aYk1g6uSM|MRYBC#3mmjJf;G?A`8+o1XSd zvz!t-2flV{W+s1R5uqryE0I;T0Vo^|R`mPYCa4$~LU6n3bgBol1~0 ze*`dV`nZoCwT_!n#Ejo%C$WHKV0k=nnF%^KN(PKxn@3TfZCJ$RQ;!zj%RjbY*!ggs z!Nl$Yfas&vRa1}#n`Q;O1cK-G(}>0_JK-PLnLt)#Lesv+WB2pTPLsMZ&{aD4If2=~ zMr{AY=H83jWK4VI@6r8yw{GI?=MP%OFC8bE6PAxhMS4p09~s#D=RvRCjCt*Yx=t^q zOf!sfmDS~fhwfzRxX6TXtIekrgKz~N_t9A))(YuwA=(xkXOA0wxCP%|7^dky_W}Tq zV;vk~wyualEqi8(`y7i9ly}x>N6hJrME%BXQn(_u29G|;hOhy@D<-Q znhnJQx4d6&@%AO!_yiJ}!LlW4XmTyb(#-yQ%Rsk=&MsjxP^JJ}uK;rQBXhu}&wyfq zS+(zTm+AyoyCq+nlr?NL_JXeWxX-U+-%_yZL`8;icN398``7Z3X1eBtMnPsvzhq#8 zL#8}}L!{}DOMGn}zmeEr5myX;tzPh{+9u%bOvuPV*DnSFiK0*CO9*`Z%TDt!5QzF$yEhhe>(Q4s&f#) z&pL4|78h6;D__Ufx-oPi&1UbbTRw9HQEw%I%H3^ZNaJn#afl7B*Y_AVEAoE@n(Oq-*=G#q~8a`bT;a;|FKSKXo+oTUcRt1eRXaq)!x0IHa4(r4Z8r}6Q`VcW*`BLRbmv04Y3rTOu zY&zaP#33RAr?Rc-kD72`3w(H-_b7*78SW zu~OaX2K{+rzIeG{R@0_^0o zsivp1NOhS}BQIjdRe1HQ0*-ThZ>uE?sgpqC_VL60;6x;Bs+Aqvth-QVHTI((Fawrq zGaaByMxXwjvAGp6`w=ky)>U8hL5~an&+O9IA?epo{zH)ZPfQ-r(Wb#qpKO7unp9m` z&;+xx>+-dihl_YzAjuP5J0^%f+3W#3#PPsBUI8xlKl~^hSn)#C)$+hC`-~fdE{pME z=<;$CA&f0<#%qAU@-hAd5D|!w!NPK*$H^^My0fx3RkDr@n4L$2&S`T6>m&%f_8laB z9DoCfM`GvlZj_LUAy4ZsRov;b8k^l@*6ZT0B>$^>ZRhKnPlP7#>z@PI0!1KPIKtwHXpEVCc}Wdz4S-CQ z@tZV96bdjhZ(Ul6`@%3_NWHu+Z8_A`wWpTXQil;E)HQCZm+~5(mY=o%YPQ|JrCztr zkt_MqICC?|#X0zJ%bX>CX!EPeCz^I)i@a)Y4ldm;7Hx_T>Q^z5k6Z>3{!sJ`AEKg; ziK>(d@J2g86L8Fm+iKO)*zypQjd8R4t|2n(Og>2ZcE!aEmmvr{hPC66cEtH4xi~A@}S@ ze~3<{S%#}?>Xk=?)%f@ z@S`iA_zCZmhdSJ6696`6aK!y4`Byu{>~d)K396u}T~5j<9)Bacl@^ zKD9;?H6UB;1V%k;%*~00!t2=3tlfG1ZaLn|Zgz-g1OnqTs6mSu-_+Yr;(0F$L+1vB zjmlw~m!rXv4=PlL9jt24orhZ&NvNHYmr9d&H0I@EOs*Sg2di*nAKvWzI=%}m!qpd5 z*Su38sC$4ko19IEu``T?T!a6E9$8Dhw^d8T_{7apv%-Hs*@kNFP<`8BUTEhUEa zQHKVcW>3I% zVu?GE_1;-_3hl#akt5^VOr5|DZSjKA%?GxJY)njFQ>A5{4-dLyQAg{11KY(SH&$mV zYnJd1Z;)gnLbpJYAA8doKOw<=x5bcUlgfEs*N1-e51VE|Rf`0&kncl_tii_f9u_@G z{{bj86r3tSf@v;WPGaJN*sCO^Ee;;*dl4VC zD(D5~J~B&K;!dt_Z5On7_g#vkMvz(O$$a_M9qnSd4?K?&g!}zBAs_yEvBh}B(Pp5` zs%3kAX{sG5HSpRDS>hx-~e;W8A|yEuF@NO7w@QSc>Pek_<)GZV%gETuFr9P*L0grLw*0 zbFNbWtn~s%fOH~GkAAQ9t)|=ZdDiSIlCzATB;yuHOnsM~6sht#>P>8y`)gC+3<9|$ zNksA99xsfmU}B)fg2(|0EF^ECQ)C*gu4F}G5J7ATB=P|kgsjB)!{sK0?AwJRG>V`Y ztuc_P$zppBP>#?nJvd&o2cV*3H@CYJ_T&ZeDZdu&Aad-9cz%~WyGG0LILWeL2jP!q zbHwEQ?oxZOFE4NvnFVuoJb=pcg%iuF8HiN^DMw6diIqQFL*cgf=Av-Eg1XyG46j=) z0^dqgJtTJ?ig*VmvXPWdX<m7LBy+WzI$hm zF_}#w#*@}dC0)?|OA`r_tsU&BdtrxzYsrS6ISiXT*MZ%E>ZXls1(be-VJoTbY_Vp~ zlVE<4KjRdNRSVQTVQ5)pzG)Z+7YOPLM6YO_w(IzRMZtSz`B-Zt?iF{;KrweCFfNx8zv)+cL#skF`1CQcKnr z{J~*#7(hTjJ@xX9npu!MT{4ADf5$0g^?xL=>D55S;kUM-=gpsCNbL6UJ-gUrXGqr! z?MQE2-g3z!Gb42~rIOymMewmP+B)&3#-zxDs*yS7;yN{uUnZtq3+PjI#rdM^NSHhb zGnLjN_wIY@%@$PWA9b$eQ)J6wQ4${J1fE^S0m>c4fh{-FZ)6)6vdXLRjiPs&r~Mho z5{Vo8z+eeb4jRbh$bX;(0{(H6%xPncv)nbpkB?SpP^TRZK_(a|6 z7g5KC&#&*@O-+c4$UBUdFKg(@GBWQc@`3qH3JU<3`V=3$_Z|O(=-snWkHfbG{HvDE zJ(uPg*4r!n(QA=dHwuCeSm()O?VRIs>BF;AKu5r(PnH47CF^u>yz+Y$l~-nZAqjT( zN?Uxm+Tpe+?5I{4des7kcN62nRjKA-#LoE~xzbZ|6iCRx7@iHjHU617NklnXAZYGT zAB6h>be0!oT1rMF$TFOyVenrm#@;*Y_ScnZ!6JA+Nt7D4j1k*6M zkRC)4sFj=S?FPH#c&a6e4rT%~lcxodaEP$CmG>?FDg$B8uW(3Pb^dVR-R*~%ppf5V zvf*_U?_OdIgb59-ydU5jSm~JNXL%J-dk7%j{{+}}6}FJQ;n(kUUcO6?hPw86!#75d zH^Rgc8b#6G{JA0^3V|VDut6kafOQk952{e`I^VB4F|pS|F~7Yu5pnOPWdEKWNXU&d zH6{GQkdx(jDO4NrdZe!o93Fu=CIyPBp zSq{24A3*X4D8NQcLPuYF{eKiNjMH?R^?(9N^-<0qH9LHnI4%D3Q=h%pWtxTkK-jtK z#EBE0Z;=SgnF6zeZH)N#wTEjB1b#LpK?}p~wZwOYVE4cqytxb|I|e;qSt8M&){BKe z&EEP%vD31TUBcKvip|Y+c01aa-3T}x6ZA`O$Y8o390E~_k_xh4yQj!G9(@+7TfBl8 zpr*=A4pf+PJS^5qZv_d|k+gcX)xt2_@a_dw_@fAaN13F18ttR*wczHm6iGCH8`K29*=tAGNE1Unn&b#V{et}{7H>Fni$Wx$FymyIDuJ6ZHrpbGfxkJ zg6D4Eez+;5#YnC4v z2G9)6-Y_J~VrC(uID>N)E|8N+c(SYV=y+8e-oW;!$02KkZ?Wx2j(j@T zXC_+Fb8;2mWB7g~;7Q?B@VP$?f9qxPQ*vc^Lzn0$ z-w9?nhkpA$R(`2;)xA;%B?ulGrA8#kjzf-OUiQ>>aTH_}ohn1Jq)(((Kcd9Aukff$}wW zG=ROo*{z;G?8Fn zCr+W1aXW}h&tkkT1imE5-bD0ofLVr_wT|RK(Ii=a4Vmci#$3POA7k`+UrEUlgMBlW z?hl%hZ2h1nVdxI;{XNU!RrS*kn{TC0K{&h8^7PHN&3<&H`7L$h!R>qzKuC%sH+-+eLC9Z#Y^ln)O9&K9`z z+5oJn=w{19)`#Zr!&kh}m0eCSXF%PCIdgHiefmA7VsVp0cdUjM4v=SsoYb*iCc~an zwLpn{xy9S3PviQUDw><@mNjT-6d`m_T-`tFlt7>de=d0nUr}Bv?QMMkjZxAJYWs;zSH5Szl2 zY@32}3wO6R0O1*HA;A2TTV{^UV1meNv|4`c&h#qM#@@P?8V=X|uDK6ttU_?I^GsY< z+;pq?QFdsM+)nx#CXRg{<4zu{c$@_I#_2lHe{*LieCDFfXSsD|z_*(VLW7BqdBU{T@WnO! z)xS1JzgMKVlDsPyIR~5t4A@Wx$Ko#wHQMUwV0S!xd%ytO@-DJU%ugd;IW9EnSHm=)^F?- z>8A=l6CmqD!Og*{8es9nbBL4)yevg&vVm)|*S7HS|H;&HiF%hnuk7m&=U{!#tsvBl8=M8-17SPmH2 zoWky%pBgMRyl&KoOSydX61*dBZB(tofTpwJAfF=%>vmx^G{`lGl4ksl6(nvZHdsDW z&}PQTdJ_Ho{R<0Q^8FDZ9jh}4b|-Oc5`?+;(v+uDd&?;j7{)?qaXW5IPv=f~^j*G_ z`f2I91*3m!x?prObru=alr(g%R zLFh6RwAt_SE}fwVcCPs|0Y7qnL?a&A9_cD;^p$=ire9=WhL%D;zorlB>T^MjO;b;^ z^GD9e`_DJ_X857jd-wu2WTUQ8A%gotVpM$ZAXAN5a>mRphG*7dn z0DC=pjpek(@%@h#LVo;D^x?Nb)zU6tirPU5E*%%APAN*2>ho~1t1=jlt zV~(|U$y~1NM}6RqONCg6Vp%WP@f9a)KN0Nd~j|!Hxq%(a6yX| zKz8`zXaZSYO&VOI>$dS*gVQ$Z-c1rhnCRVzW)_g|#bt@#yag;h zgN_?`7gZ!ROLVil4lp1MtUE8(Vp14o>f8J=jR2H0)}}5;f{yAmjo0bgz`rJX8%>of zhRfwZZ)f5)?t#zXBphT-=x^^D%_6DU``zI5**s0Vz&EO)8om+u7Ge<>=am)(e3BJI z1qAUcJxi3dAt&1RENUj5lG009ncVn&@{cz4`_vqt<6!iH2ya2e0t=?C687kIipMaZh z3Suy7$K*1BM{Et3*cv_8eo+s7^!c?LZMN1!4N-?^CXo, with _2, _3, ... appended if that already exists. Never next to + //! ~/Documents/Easy Forest HDMapping/, with _2, _3, ... appended if that already exists. Never next to //! the recording: that may be the scanner's USB stick. fs::path defaultOutputFolder(const fs::path& recording) { - const fs::path base = homeFolder() / "Documents" / "HDMapping Easy Forestry"; + const fs::path base = homeFolder() / "Documents" / "Easy Forest HDMapping"; const std::string name = recording.filename().empty() ? recording.parent_path().filename().string() : recording.filename().string(); std::error_code ec; fs::path candidate = base / name; @@ -572,7 +572,7 @@ int main(int argc, char* argv[]) app.setRecording(argv[first_positional]); SetConfigFlags(FLAG_WINDOW_RESIZABLE | FLAG_MSAA_4X_HINT); - InitWindow(900, 780, ("HDMapping Easy Forestry " + std::string(HDMAPPING_CONFIGURE_PROJECT_VERSION)).c_str()); + InitWindow(900, 780, ("Easy Forest HDMapping " + std::string(HDMAPPING_CONFIGURE_PROJECT_VERSION)).c_str()); raylib_widgets::fitWindowToScreen(); // raylib's default exit key (Esc) would close the window mid-run. SetExitKey(KEY_NULL); @@ -605,9 +605,9 @@ int main(int argc, char* argv[]) ImGui::SetNextWindowPos(viewport->WorkPos); ImGui::SetNextWindowSize(viewport->WorkSize); ImGui::Begin( - "HDMapping Easy Forestry", nullptr, ImGuiWindowFlags_NoDecoration | ImGuiWindowFlags_NoMove | ImGuiWindowFlags_NoSavedSettings); + "Easy Forest HDMapping", nullptr, ImGuiWindowFlags_NoDecoration | ImGuiWindowFlags_NoMove | ImGuiWindowFlags_NoSavedSettings); - ImGui::TextUnformatted("HDMapping Easy Forestry"); + ImGui::TextUnformatted("Easy Forest HDMapping"); ImGui::TextDisabled("Mandeye recording -> lidar odometry (precision forestry settings) -> point cloud and trajectory"); ImGui::Spacing(); diff --git a/apps/easy_gui/src/export.cpp b/apps/easy_gui/src/export.cpp index 983ab72a..a9777d3f 100644 --- a/apps/easy_gui/src/export.cpp +++ b/apps/easy_gui/src/export.cpp @@ -210,9 +210,9 @@ namespace easy_gui std::snprintf(near_min, sizeof(near_min), "%.2f m", request.near_min_range_m); std::snprintf(output_filter, sizeof(output_filter), "%.2f m", stats.threshould_output_filter); - out << "HDMapping Easy Forestry results\n" + out << "Easy Forest HDMapping results\n" << "==========================\n\n" - << "Made by HDMapping Easy Forestry " << HDMAPPING_CONFIGURE_PROJECT_VERSION << " from the recording\n" + << "Made by Easy Forest HDMapping " << HDMAPPING_CONFIGURE_PROJECT_VERSION << " from the recording\n" << " " << request.recording.laz_files.front().parent_path().string() << "\n" << "with lidar odometry (precision forestry settings). No loop closure and no georeferencing: coordinates\n" << "are in metres in a local frame that starts at the scanner's first position (z up, from the IMU).\n\n" @@ -248,7 +248,7 @@ namespace easy_gui << " point_origin extra bytes, uint8: where the point comes from (see below)\n" << (request.place_by_raw_time ? " 0 = inside lidar odometry's distance limits, placed by raw time\n" : " 0 = lidar odometry's own point\n") - << " 1 = near point, 2 = gap point (both added by Easy Forestry)\n" + << " 1 = near point, 2 = gap point (both added by Easy Forest HDMapping)\n" << (stats.check_band_points ? " 3 = TEST MODE alignment-check point (duplicate of an odometry point)\n" : "") << "\n" @@ -257,7 +257,7 @@ namespace easy_gui << "frame) is between " << inner << " and " << outer << " and whose 3D distance is above " << output_filter << ", and it works in\n" << "0.1 s pieces: it skips the raw points measured after the last pose of a piece and before the first pose\n" - << "of the next one (about 5 % of the recording time). Easy Forestry adds both kinds of missing points back from\n" + << "of the next one (about 5 % of the recording time). Easy Forest HDMapping adds both kinds of missing points back from\n" << "the raw recording, so the cloud has no hole around the walking path and nothing is missing between the\n" << "pieces.\n\n" << "point_origin = 0 (" << formatCount(stats.odometry_points) << " points): " diff --git a/apps/easy_gui/src/las14_writer.cpp b/apps/easy_gui/src/las14_writer.cpp index ab1da4bb..a8dc325a 100644 --- a/apps/easy_gui/src/las14_writer.cpp +++ b/apps/easy_gui/src/las14_writer.cpp @@ -89,7 +89,7 @@ namespace easy_gui header->global_encoding = (1 << 4); // bit 0 = 0: not adjusted GPS time; bit 4: WKT (required for PF6) header->version_major = 1; header->version_minor = 4; - copyField(header->system_identifier, sizeof(header->system_identifier), "HDMapping Easy Forestry"); + copyField(header->system_identifier, sizeof(header->system_identifier), "Easy Forest HDMapping"); copyField(header->generating_software, sizeof(header->generating_software), generating_software); { const std::time_t now = std::time(nullptr); diff --git a/apps/easy_gui/src/pipeline.cpp b/apps/easy_gui/src/pipeline.cpp index e7e9d2fd..6b070c8b 100644 --- a/apps/easy_gui/src/pipeline.cpp +++ b/apps/easy_gui/src/pipeline.cpp @@ -209,7 +209,7 @@ namespace easy_gui } log_file_.open(out.log_file, std::ios::out | std::ios::app); - log("[easy_gui] HDMapping Easy Forestry " + std::string(HDMAPPING_CONFIGURE_PROJECT_VERSION) + ", started " + timestamp()); + log("[easy_gui] Easy Forest HDMapping " + std::string(HDMAPPING_CONFIGURE_PROJECT_VERSION) + ", started " + timestamp()); log("[easy_gui] recording: " + request_.recording.string()); log("[easy_gui] output: " + out.root.string()); log("[easy_gui] step 1 program: " + request_.step1_executable.string()); @@ -249,8 +249,8 @@ namespace easy_gui cleanupWork(false); finish( PipelineState::Failed, - "Easy Forestry could not find the lidar odometry program (lidar_odometry_step_1) next to itself. Please reinstall " - "HDMapping Easy Forestry.", + "Easy Forest HDMapping could not find the lidar odometry program (lidar_odometry_step_1) next to itself. Please reinstall " + "Easy Forest HDMapping.", "looked for: " + request_.step1_executable.string()); busy_ = false; return; diff --git a/apps/easy_gui/src/preflight.cpp b/apps/easy_gui/src/preflight.cpp index b2b93d17..3dab021a 100644 --- a/apps/easy_gui/src/preflight.cpp +++ b/apps/easy_gui/src/preflight.cpp @@ -358,7 +358,7 @@ namespace easy_gui ids += (ids.empty() ? "" : ", ") + std::to_string(id); add(Severity::Error, "This recording contains data from more than one lidar (ids " + ids + - "). Easy Forestry supports single-lidar recordings only; use the standard HDMapping apps for this one."); + "). Easy Forest HDMapping supports single-lidar recordings only; use the standard HDMapping apps for this one."); } else if (lidar_ids.size() == 1) r.lidar_id = *lidar_ids.begin(); @@ -375,7 +375,7 @@ namespace easy_gui else if (calibration.size() > 1) add(Severity::Error, "The calibration file '" + f.calibration_file.filename().string() + - "' describes more than one lidar. Easy Forestry supports single-lidar recordings only."); + "' describes more than one lidar. Easy Forest HDMapping supports single-lidar recordings only."); else { for (const auto& [id, sn] : id_to_sn) @@ -391,7 +391,7 @@ namespace easy_gui } if (id_to_sn.size() > 1) add(Severity::Error, - "The scanner serial-number file lists more than one lidar. Easy Forestry supports single-lidar recordings only."); + "The scanner serial-number file lists more than one lidar. Easy Forest HDMapping supports single-lidar recordings only."); // --- IMU vs lidar clocks double imu_first = 0.0, imu_last = 0.0; @@ -426,7 +426,7 @@ namespace easy_gui add(Severity::Error, "Choose an output folder."); else if (isSameOrInside(output_folder, recording)) add(Severity::Error, - "The output folder is inside the recording folder. Choose a folder elsewhere: Easy Forestry never writes into the " + "The output folder is inside the recording folder. Choose a folder elsewhere: Easy Forest HDMapping never writes into the " "recording " "(it may be on the scanner's USB stick)."); else From 86ae366c3c322bca8bc1cc965b7e51dde433d9e2 Mon Sep 17 00:00:00 2001 From: Martin Mokros Date: Sun, 18 Oct 2026 12:41:04 +0100 Subject: [PATCH 11/23] Easy Forest HDMapping: macOS and Windows release workflows New workflows easy-forest-macos.yml / easy-forest-windows.yml (tags easy-forest-v*, manual dispatch), separate from the upstream ones, building with HDMAPPING_EASY_GUI_ONLY=ON. Packaging scripts in apps/easy_gui/packaging: - macOS: static PROJ built for 13.0, oneTBB without hwloc, .app bundle with easy_gui + lidar_odometry_step_1 + dylibs (rpaths rewritten), ad hoc re-sign, minos <= 13.0 check on every bundled binary, DMG with an Applications shortcut. - Windows: easy_gui is a WIN32-subsystem app (CLI modes attach to the parent console), UTF-8 process code page manifest, folder + zip with all DLLs and the MSVC runtime, checked with dumpbin. HDMAPPING_EASY_GUI_ONLY builds no longer configure OpenCV. easy_gui_tests now also cover ChildProcess on Windows and argument quoting on all platforms. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/easy-forest-macos.yml | 127 ++++++++++++ .github/workflows/easy-forest-windows.yml | 102 ++++++++++ apps/easy_gui/CMakeLists.txt | 9 + apps/easy_gui/docs/DESIGN.md | 31 +-- apps/easy_gui/packaging/macos/build_proj.sh | 44 +++++ apps/easy_gui/packaging/macos/make_app.py | 208 ++++++++++++++++++++ apps/easy_gui/packaging/macos/make_dmg.sh | 20 ++ apps/easy_gui/packaging/windows/make_zip.py | 128 ++++++++++++ apps/easy_gui/src/child_process.cpp | 14 ++ apps/easy_gui/src/child_process.h | 5 + apps/easy_gui/src/easy_gui.manifest | 11 ++ apps/easy_gui/src/easy_gui_main.cpp | 2 + apps/easy_gui/tests/test_child_process.cpp | 104 +++++++++- apps/easy_gui/tests/test_main.cpp | 20 +- cmake/dependencies.cmake | 10 +- 15 files changed, 818 insertions(+), 17 deletions(-) create mode 100644 .github/workflows/easy-forest-macos.yml create mode 100644 .github/workflows/easy-forest-windows.yml create mode 100755 apps/easy_gui/packaging/macos/build_proj.sh create mode 100755 apps/easy_gui/packaging/macos/make_app.py create mode 100755 apps/easy_gui/packaging/macos/make_dmg.sh create mode 100755 apps/easy_gui/packaging/windows/make_zip.py create mode 100644 apps/easy_gui/src/easy_gui.manifest diff --git a/.github/workflows/easy-forest-macos.yml b/.github/workflows/easy-forest-macos.yml new file mode 100644 index 00000000..77f4ede1 --- /dev/null +++ b/.github/workflows/easy-forest-macos.yml @@ -0,0 +1,127 @@ +# Easy Forest HDMapping release build for macOS (Apple silicon, macOS 13.0 and newer). +# Separate from the upstream macos.yml: builds only easy_gui + lidar_odometry_step_1 (HDMAPPING_EASY_GUI_ONLY) and +# packages them as "Easy Forest HDMapping.app" in a DMG. The packaging steps are scripts in +# apps/easy_gui/packaging/macos, so they can be run and debugged locally the same way. +name: Easy Forest HDMapping macOS + +on: + push: + tags: ["easy-forest-v*"] + workflow_dispatch: + +permissions: + contents: write # attach the DMG to the release on a tag + +env: + MIN_MACOS: "13.0" + # Picked up by every CMake project (PROJ, the FetchContent raylib) and by clang directly. + MACOSX_DEPLOYMENT_TARGET: "13.0" + +jobs: + build: + runs-on: macos-15 + + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + fetch-depth: 0 + + - name: Set version + run: | + if [[ "$GITHUB_REF" == refs/tags/easy-forest-v* ]]; then + echo "VERSION=${GITHUB_REF_NAME#easy-forest-v}" >> "$GITHUB_ENV" + else + echo "VERSION=$(git describe --tags --always)" >> "$GITHUB_ENV" + fi + + - uses: lukka/get-cmake@latest + with: + cmakeVersion: "~4.0.0" + + # Homebrew's PROJ is built for the runner's macOS; build a static one for MIN_MACOS instead (see the script). + - name: Cache PROJ + id: proj-cache + uses: actions/cache@v4 + with: + path: ${{ runner.temp }}/proj-install + key: proj-macos-${{ env.MIN_MACOS }}-${{ runner.arch }}-${{ hashFiles('apps/easy_gui/packaging/macos/build_proj.sh') }} + + - name: Build PROJ + if: steps.proj-cache.outputs.cache-hit != 'true' + run: apps/easy_gui/packaging/macos/build_proj.sh "$RUNNER_TEMP/proj-install" "$RUNNER_TEMP/proj-work" + + # TBB_DISABLE_HWLOC_AUTOMATIC_SEARCH: the bundled oneTBB would otherwise pick up a Homebrew hwloc for tbbbind. + - name: Configure + run: | + cmake -B build \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_OSX_DEPLOYMENT_TARGET="$MIN_MACOS" \ + -DHDMAPPING_EASY_GUI_ONLY=ON \ + -DBUILD_TESTING=ON \ + -DFREEGLUT_COCOA=ON \ + -DHD_CPU_OPTIMIZATION=ARM \ + -DTBB_DISABLE_HWLOC_AUTOMATIC_SEARCH=ON \ + -DPROJ_DIR="$RUNNER_TEMP/proj-install/lib/cmake/proj" + + # A static library built for a newer macOS only shows up as a linker warning (the executables still say + # minos 13.0), so treat that warning as an error. + - name: Build + run: | + set -o pipefail + cmake --build build --config Release -j"$(sysctl -n hw.ncpu)" 2>&1 | tee build.log + if grep -i "built for newer macOS version" build.log; then + echo "::error::Something was linked that was built for a newer macOS than $MIN_MACOS" + exit 1 + fi + + - name: Test + working-directory: build + run: ctest -C Release --output-on-failure + + # Also checks minos <= MIN_MACOS for every bundled binary and fails otherwise. + - name: Create app bundle + run: python3 apps/easy_gui/packaging/macos/make_app.py build/bin dist --version "$VERSION" --min-macos "$MIN_MACOS" + + - name: Smoke test the bundle + run: | + set +e + "dist/Easy Forest HDMapping.app/Contents/MacOS/easy_gui" --headless > usage.txt 2>&1 + rc=$? + set -e + cat usage.txt + # Exit code 2 + usage text = dyld loaded every bundled library and main() ran. + test "$rc" -eq 2 && grep -q "usage:" usage.txt + + - name: Create DMG + run: apps/easy_gui/packaging/macos/make_dmg.sh "dist/Easy Forest HDMapping.app" dist/EasyForestHDMapping-mac.dmg + + - name: Sizes + run: | + { + echo "### Easy Forest HDMapping macOS $VERSION" + echo '```' + ls -l dist/EasyForestHDMapping-mac.dmg + du -sh "dist/Easy Forest HDMapping.app" + find "dist/Easy Forest HDMapping.app/Contents" -type f -exec ls -l {} \; | awk '{printf "%10d %s\n", $5, substr($0, index($0, $9))}' + echo '```' + } | tee -a "$GITHUB_STEP_SUMMARY" + + - uses: actions/upload-artifact@v4 + with: + name: EasyForestHDMapping-mac-${{ env.VERSION }} + path: dist/EasyForestHDMapping-mac.dmg + compression-level: 0 # already compressed + retention-days: 90 + + # The macOS and Windows workflows both run on the same tag: whichever gets here first creates the release. + - name: Attach to GitHub release + if: startsWith(github.ref, 'refs/tags/easy-forest-v') + env: + GH_TOKEN: ${{ github.token }} + run: | + gh release view "$GITHUB_REF_NAME" > /dev/null 2>&1 \ + || gh release create "$GITHUB_REF_NAME" --verify-tag --title "Easy Forest HDMapping $VERSION" \ + --notes "Installation and use: [apps/easy_gui/README.md](https://github.com/${GITHUB_REPOSITORY}/blob/${GITHUB_REF_NAME}/apps/easy_gui/README.md)" \ + || gh release view "$GITHUB_REF_NAME" > /dev/null + gh release upload "$GITHUB_REF_NAME" dist/EasyForestHDMapping-mac.dmg --clobber diff --git a/.github/workflows/easy-forest-windows.yml b/.github/workflows/easy-forest-windows.yml new file mode 100644 index 00000000..8c912673 --- /dev/null +++ b/.github/workflows/easy-forest-windows.yml @@ -0,0 +1,102 @@ +# Easy Forest HDMapping release build for Windows (x64). +# Separate from the upstream windows.yml: builds only easy_gui + lidar_odometry_step_1 (HDMAPPING_EASY_GUI_ONLY) and +# packages them with every DLL they need, the MSVC runtime included, as one zip (apps/easy_gui/packaging/windows). +name: Easy Forest HDMapping Windows + +on: + push: + tags: ["easy-forest-v*"] + workflow_dispatch: + +permissions: + contents: write # attach the zip to the release on a tag + +jobs: + build: + runs-on: windows-latest + + defaults: + run: + shell: bash + + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + fetch-depth: 0 + + - name: Set version + run: | + if [[ "$GITHUB_REF" == refs/tags/easy-forest-v* ]]; then + echo "VERSION=${GITHUB_REF_NAME#easy-forest-v}" >> "$GITHUB_ENV" + else + echo "VERSION=$(git describe --tags --always)" >> "$GITHUB_ENV" + fi + + - uses: lukka/get-cmake@latest + with: + cmakeVersion: "~4.0.0" + + # HD_CPU_OPTIMIZATION=GENERIC: AUTO adds /arch:AVX2 when the runner's CPU has it, and the release must also run + # on PCs without AVX2. + - name: Configure + run: | + cmake -B build -G "Visual Studio 17 2022" -A x64 \ + -DHDMAPPING_EASY_GUI_ONLY=ON \ + -DBUILD_TESTING=ON \ + -DHD_CPU_OPTIMIZATION=GENERIC \ + -DTBB_DISABLE_HWLOC_AUTOMATIC_SEARCH=ON + + - name: Build + run: cmake --build build --config Release --parallel + + - name: Test + working-directory: build + run: ctest -C Release --output-on-failure + + - name: Create folder and zip + run: | + python apps/easy_gui/packaging/windows/make_zip.py build/bin/Release \ + "dist/Easy Forest HDMapping" dist/EasyForestHDMapping-windows.zip \ + --search-dir build/3rd_binary/PROJ/install_proj/bin + + - name: Smoke test the folder + run: | + # easy_gui.exe is a windowed program, so check from its own folder that it and lidar_odometry_step_1.exe + # load all their DLLs: run with bad arguments and expect the usage exit code 2. + set +e + "dist/Easy Forest HDMapping/easy_gui.exe" --headless + rc=$? + set -e + echo "easy_gui.exe exit code: $rc" + test "$rc" -eq 2 + + - name: Sizes + run: | + { + echo "### Easy Forest HDMapping Windows $VERSION" + echo '```' + ls -l dist/EasyForestHDMapping-windows.zip + du -sh "dist/Easy Forest HDMapping" + ls -l "dist/Easy Forest HDMapping" + echo '```' + } | tee -a "$GITHUB_STEP_SUMMARY" + + - uses: actions/upload-artifact@v4 + with: + name: EasyForestHDMapping-windows-${{ env.VERSION }} + path: dist/EasyForestHDMapping-windows.zip + compression-level: 0 # already compressed + retention-days: 90 + + # The macOS and Windows workflows both run on the same tag: whichever gets here first creates the release. + - name: Attach to GitHub release + if: startsWith(github.ref, 'refs/tags/easy-forest-v') + env: + GH_TOKEN: ${{ github.token }} + run: | + gh release view "$GITHUB_REF_NAME" > /dev/null 2>&1 \ + || gh release create "$GITHUB_REF_NAME" --verify-tag --title "Easy Forest HDMapping $VERSION" \ + --notes "Installation and use: [apps/easy_gui/README.md](https://github.com/${GITHUB_REPOSITORY}/blob/${GITHUB_REF_NAME}/apps/easy_gui/README.md)" \ + || gh release view "$GITHUB_REF_NAME" > /dev/null + gh release upload "$GITHUB_REF_NAME" dist/EasyForestHDMapping-windows.zip --clobber diff --git a/apps/easy_gui/CMakeLists.txt b/apps/easy_gui/CMakeLists.txt index e9555d99..ffe412bc 100644 --- a/apps/easy_gui/CMakeLists.txt +++ b/apps/easy_gui/CMakeLists.txt @@ -87,6 +87,15 @@ target_link_libraries(easy_gui PRIVATE add_dependencies(easy_gui lidar_odometry_step_1) if(WIN32) + # A windowed program: no console window behind the GUI. main() stays the entry point (the WIN32 subsystem would + # otherwise expect WinMain); --headless/--export-only print into the console they were started from + # (attachToParentConsole). The manifest makes UTF-8 the process code page, see src/easy_gui.manifest. + set_target_properties(easy_gui PROPERTIES WIN32_EXECUTABLE ON) + if(MSVC) + target_link_options(easy_gui PRIVATE /ENTRY:mainCRTStartup) + endif() + target_sources(easy_gui PRIVATE src/easy_gui.manifest) + add_custom_command( TARGET easy_gui POST_BUILD diff --git a/apps/easy_gui/docs/DESIGN.md b/apps/easy_gui/docs/DESIGN.md index e3d65c4a..847f501b 100644 --- a/apps/easy_gui/docs/DESIGN.md +++ b/apps/easy_gui/docs/DESIGN.md @@ -400,19 +400,26 @@ always reports the breakdown, warns below 1.5ร— the estimate and refuses below 1 gps_time, reported twice: as step 1 does it (no-return points removed by the load filter *before* the ramp) and with those points kept, to quantify how much their removal contributes. - Near points: counts, range histogram, and sensor-frame density below 1.5 m. -- **Windows:** `child_process.cpp` has a Windows implementation (CreateProcess + job object) that has not been compiled - yet. - **v2:** gated pose-graph loop closure (ยง1). -- **Packaging (macOS, not done yet):** - - Bundle `easy_gui` and `lidar_odometry_step_1` in one `Easy Forest HDMapping.app` (step 1 must stay next to - `easy_gui`), with their dylibs, and rewrite the library paths. - - Then re-sign the whole bundle ad hoc, because rewriting library paths invalidates the signatures and Apple silicon - refuses to run unsigned code: `codesign --force --deep -s - "Easy Forest HDMapping.app"`. This must include the - bundled step 1 binary and every bundled dylib. - - Every bundled dylib must be built for macOS 13 too, or the app's real minimum is higher than the README says. A - build against Homebrew links `libproj` (into both programs) and `libhwloc` (through TBB), both built for the build - machine's macOS (26 here; the linker warns). Build them from source with `MACOSX_DEPLOYMENT_TARGET=13.0`, or drop - them from Easy GUI's link, and check every file with `vtool -show-build` (`minos`). +- **Release builds:** `.github/workflows/easy-forest-{macos,windows}.yml`, on `easy-forest-v*` tags and manual + dispatch; the packaging steps are scripts in `apps/easy_gui/packaging/` so they run locally the same way. + - macOS (`macos-15` runner, Apple silicon, minimum 13.0): PROJ is built from source as a static library for 13.0 + (`build_proj.sh`; core_math links it for GNSS, which Easy GUI never calls; Homebrew's is built for the runner's + macOS and drags in libtiff/curl). oneTBB without hwloc (`TBB_DISABLE_HWLOC_AUTOMATIC_SEARCH=ON`). OpenCV is not + configured at all in `HDMAPPING_EASY_GUI_ONLY` builds (only the camera apps link it). `make_app.py` builds + `Easy Forest HDMapping.app` (`easy_gui` main executable, `lidar_odometry_step_1` next to it in `Contents/MacOS`, + `liblaszip`, `libtbb`, `libglut` in `Contents/Frameworks`, rpaths rewritten), re-signs it ad hoc and fails unless + every bundled binary has minos <= 13.0 and references nothing outside the bundle, `/usr/lib` and `/System`. The + build also fails on the linker's "built for newer macOS version" warning, which a static library would only show + there. `make_dmg.sh` adds the Applications shortcut. Local run: app 7.9 MB, DMG 3.5 MB; the bundled app processed + `mini4` headless. + - Windows (`windows-latest`, x64): `easy_gui.exe` is a WIN32-subsystem program (`/ENTRY:mainCRTStartup`, so `main` + stays), `--headless`/`--export-only` attach to the parent console. A manifest sets UTF-8 as the process code page, + because the file dialog, drag and drop and `ChildProcess` all use UTF-8. `HD_CPU_OPTIMIZATION=GENERIC`: `AUTO` + adds `/arch:AVX2` from the runner's CPU. `make_zip.py` collects the DLLs with `dumpbin /dependents`, copies the + MSVC runtime app-locally from VS's redist folder and fails on any DLL that is neither bundled nor part of Windows. + - `easy_gui_tests` cover `ChildProcess` on Windows too (cmd.exe; argument quoting through `--echo-args`). + - No app icon yet (none exists for Easy GUI); no code signing or notarization (README explains the warnings). Tests: `easy_gui_tests` (doctest, synthetic recordings and sessions). The integration tests against the real step 1 binary run only when `EASY_GUI_TEST_RECORDING=` is set; a folder of symlinks to the first 4 file diff --git a/apps/easy_gui/packaging/macos/build_proj.sh b/apps/easy_gui/packaging/macos/build_proj.sh new file mode 100755 index 00000000..d7c5dc86 --- /dev/null +++ b/apps/easy_gui/packaging/macos/build_proj.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# Builds a static PROJ for the Easy Forest HDMapping macOS release. +# +# Usage: build_proj.sh [work dir] +# +# Why: core_math links PROJ::proj (core/src/gnss.cpp, GNSS georeferencing, which Easy GUI never calls), so every +# build needs one. Homebrew's libproj is built for the runner's own macOS version (and pulls in libtiff/curl/sqlite +# dylibs built the same way), so a bundle using it would not start on older macOS. A static PROJ built here for the +# deployment target leaves nothing to bundle: only the system libsqlite3 and libc++ are linked dynamically. +# TIFF (grids) and curl (network grids) are off: neither is needed to compile/link core_math. +set -euo pipefail + +PROJ_VERSION=9.6.2 +PROJ_SHA256=53d0cafaee3bb2390264a38668ed31d90787de05e71378ad7a8f35bb34c575d1 +DEPLOYMENT_TARGET="${MACOSX_DEPLOYMENT_TARGET:-13.0}" + +prefix="${1:?usage: build_proj.sh [work dir]}" +work="${2:-$(mktemp -d)}" +mkdir -p "$work" +prefix="$(cd "$(dirname "$prefix")" && pwd)/$(basename "$prefix")" + +tarball="$work/proj-$PROJ_VERSION.tar.gz" +if [ ! -f "$tarball" ]; then + curl -sSLf -o "$tarball" "https://github.com/OSGeo/PROJ/releases/download/$PROJ_VERSION/proj-$PROJ_VERSION.tar.gz" +fi +echo "$PROJ_SHA256 $tarball" | shasum -a 256 -c - + +rm -rf "$work/proj-$PROJ_VERSION" "$work/build" +tar -xzf "$tarball" -C "$work" + +cmake -S "$work/proj-$PROJ_VERSION" -B "$work/build" \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX="$prefix" \ + -DCMAKE_OSX_DEPLOYMENT_TARGET="$DEPLOYMENT_TARGET" \ + -DBUILD_SHARED_LIBS=OFF \ + -DBUILD_TESTING=OFF \ + -DBUILD_APPS=OFF \ + -DENABLE_TIFF=OFF \ + -DENABLE_CURL=OFF \ + -DEMBED_PROJ_DATA_PATH=OFF +cmake --build "$work/build" --config Release -j"$(sysctl -n hw.ncpu)" +cmake --install "$work/build" --config Release + +echo "PROJ $PROJ_VERSION installed to $prefix (macOS $DEPLOYMENT_TARGET, static)" diff --git a/apps/easy_gui/packaging/macos/make_app.py b/apps/easy_gui/packaging/macos/make_app.py new file mode 100755 index 00000000..62228d13 --- /dev/null +++ b/apps/easy_gui/packaging/macos/make_app.py @@ -0,0 +1,208 @@ +#!/usr/bin/env python3 +"""Builds "Easy Forest HDMapping.app" from a build/bin directory. + +Usage: make_app.py [--version X.Y.Z] [--min-macos 13.0] + +Layout: + Contents/MacOS/easy_gui main executable (CFBundleExecutable) + Contents/MacOS/lidar_odometry_step_1 started by easy_gui as a child process from its own directory + Contents/Frameworks/*.dylib every non-system dylib either executable needs, recursively + +Every bundled Mach-O refers to its bundled dependencies as @rpath/; the executables get the single rpath +@executable_path/../Frameworks and the dylibs @loader_path, so nothing points back into the build machine. The bundle +is then re-signed ad hoc (path rewriting invalidates the linker's signature, and arm64 refuses to run unsigned code) +and checked: every bundled binary must have minos <= --min-macos and may only reference the bundle, /usr/lib and +/System. Any violation fails the script, so a CI build cannot ship a bundle that does not start on the target macOS. +""" + +import argparse +import os +import plistlib +import re +import shutil +import subprocess +import sys +from pathlib import Path + +APP_NAME = "Easy Forest HDMapping" +BUNDLE_ID = "io.github.martinmokros.easyforesthdmapping" +EXECUTABLES = ["easy_gui", "lidar_odometry_step_1"] +SYSTEM_PREFIXES = ("/usr/lib/", "/System/") + + +def run(*args): + return subprocess.run(args, check=True, capture_output=True, text=True).stdout + + +def is_macho(path): + with open(path, "rb") as f: + magic = f.read(4) + return magic in (b"\xcf\xfa\xed\xfe", b"\xce\xfa\xed\xfe", b"\xca\xfe\xba\xbe") + + +def load_commands(path): + """(dependencies, rpaths, dylib id, minimum macOS) from otool -l.""" + deps, rpaths, dylib_id, minos = [], [], None, None + cmd = None + for line in run("otool", "-l", str(path)).splitlines(): + line = line.strip() + if line.startswith("cmd "): + cmd = line.split()[1] + elif line.startswith("name ") and cmd in ("LC_LOAD_DYLIB", "LC_LOAD_WEAK_DYLIB", "LC_REEXPORT_DYLIB"): + deps.append(line[5:].rsplit(" (offset", 1)[0]) + elif line.startswith("name ") and cmd == "LC_ID_DYLIB": + dylib_id = line[5:].rsplit(" (offset", 1)[0] + elif line.startswith("path ") and cmd == "LC_RPATH": + rpaths.append(line[5:].rsplit(" (offset", 1)[0]) + elif line.startswith("minos ") and cmd == "LC_BUILD_VERSION": + minos = line.split()[1] + elif line.startswith("version ") and cmd == "LC_VERSION_MIN_MACOSX": + minos = line.split()[1] + return deps, rpaths, dylib_id, minos + + +def expand(path, loader, executable): + return Path(path.replace("@loader_path", str(loader.parent)).replace("@executable_path", str(executable.parent))) + + +def resolve(dep, macho, rpaths, executable): + """Where the build-tree binary `macho` really loads `dep` from.""" + if dep.startswith("@rpath/"): + for rpath in rpaths: + candidate = expand(rpath, macho, executable) / dep[len("@rpath/"):] + if candidate.exists(): + return candidate.resolve() + return None + candidate = expand(dep, macho, executable) + return candidate.resolve() if candidate.exists() else None + + +def version_tuple(v): + return tuple(int(x) for x in v.split(".")) + + +def bundle(bin_dir, out_dir, version, min_macos): + app = out_dir / f"{APP_NAME}.app" + if app.exists(): + shutil.rmtree(app) + macos_dir = app / "Contents" / "MacOS" + frameworks = app / "Contents" / "Frameworks" + resources = app / "Contents" / "Resources" + for d in (macos_dir, frameworks, resources): + d.mkdir(parents=True) + + # Source binary -> bundled copy, walked breadth first so each dylib is resolved with the rpaths of the build-tree + # binary that loads it. + queue = [] + for name in EXECUTABLES: + src = bin_dir / name + if not src.is_file(): + sys.exit(f"error: {src} not found") + dst = macos_dir / name + shutil.copy2(src, dst) + queue.append((src, dst, src)) + + bundled = {} # resolved source path -> bundled name + while queue: + src, dst, executable = queue.pop(0) + deps, rpaths, _, _ = load_commands(src) + for dep in deps: + if dep.startswith(SYSTEM_PREFIXES): + continue + real = resolve(dep, src, rpaths, executable) + if real is None: + sys.exit(f"error: {src.name} needs {dep}, which cannot be found") + name = Path(dep).name + if real not in bundled: + if any(n == name for n in bundled.values()): + sys.exit(f"error: two different dylibs are both called {name}") + bundled[real] = name + lib_dst = frameworks / name + shutil.copy2(real, lib_dst) + lib_dst.chmod(0o755) + queue.append((real, lib_dst, executable)) + print(f" bundling {real} as Frameworks/{name}") + if dep != f"@rpath/{bundled[real]}": + run("install_name_tool", "-change", dep, f"@rpath/{bundled[real]}", str(dst)) + for rpath in rpaths: + run("install_name_tool", "-delete_rpath", rpath, str(dst)) + new_rpath = "@executable_path/../Frameworks" if dst.parent == macos_dir else "@loader_path" + run("install_name_tool", "-add_rpath", new_rpath, str(dst)) + if dst.parent == frameworks: + run("install_name_tool", "-id", f"@rpath/{dst.name}", str(dst)) + + with open(app / "Contents" / "Info.plist", "wb") as f: + plistlib.dump( + { + "CFBundleDevelopmentRegion": "en", + "CFBundleDisplayName": APP_NAME, + "CFBundleName": APP_NAME, + "CFBundleExecutable": "easy_gui", + "CFBundleIdentifier": BUNDLE_ID, + "CFBundleInfoDictionaryVersion": "6.0", + "CFBundlePackageType": "APPL", + "CFBundleShortVersionString": version, + "CFBundleVersion": version, + "LSMinimumSystemVersion": min_macos, + "LSApplicationCategoryType": "public.app-category.utilities", + "NSHighResolutionCapable": True, + "NSHumanReadableCopyright": "MIT License. Built on HDMapping by Janusz Bฤ™dkowski and the MapsHD contributors.", + }, + f, + ) + + # Only after all rewriting: install_name_tool invalidates signatures. + run("codesign", "--force", "--deep", "-s", "-", str(app)) + return app + + +def verify(app, min_macos): + errors = [] + machos = [p for p in app.rglob("*") if p.is_file() and not p.is_symlink() and is_macho(p)] + frameworks = app / "Contents" / "Frameworks" + print(f"Checking {len(machos)} Mach-O files (minos <= {min_macos}):") + for p in sorted(machos): + deps, rpaths, _, minos = load_commands(p) + rel = p.relative_to(app) + print(f" {rel}: minos {minos}") + if minos is None: + errors.append(f"{rel}: no minimum macOS version in the load commands") + elif version_tuple(minos) > version_tuple(min_macos): + errors.append(f"{rel}: minos {minos} > {min_macos}") + for rpath in rpaths: + if not rpath.startswith(("@executable_path", "@loader_path")): + errors.append(f"{rel}: rpath {rpath} points outside the bundle") + for dep in deps: + if dep.startswith(SYSTEM_PREFIXES): + continue + if not dep.startswith("@rpath/") or not (frameworks / dep[len("@rpath/"):]).is_file(): + errors.append(f"{rel}: dependency {dep} is not in Contents/Frameworks") + try: + run("codesign", "--verify", "--deep", "--strict", str(app)) + except subprocess.CalledProcessError as e: + errors.append(f"codesign --verify failed: {e.stderr.strip()}") + return errors + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("bin_dir", type=Path) + parser.add_argument("out_dir", type=Path) + parser.add_argument("--version", default="0.0.0") + parser.add_argument("--min-macos", default="13.0") + args = parser.parse_args() + + version = re.sub(r"^[^0-9]*", "", args.version) or "0.0.0" + args.out_dir.mkdir(parents=True, exist_ok=True) + app = bundle(args.bin_dir.resolve(), args.out_dir.resolve(), version, args.min_macos) + errors = verify(app, args.min_macos) + if errors: + print("\nBundle check FAILED:", file=sys.stderr) + for e in errors: + print(f" {e}", file=sys.stderr) + sys.exit(1) + print(f"\n{app} OK") + + +if __name__ == "__main__": + main() diff --git a/apps/easy_gui/packaging/macos/make_dmg.sh b/apps/easy_gui/packaging/macos/make_dmg.sh new file mode 100755 index 00000000..cec64205 --- /dev/null +++ b/apps/easy_gui/packaging/macos/make_dmg.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# Packs "Easy Forest HDMapping.app" into a DMG whose window shows the app next to an Applications shortcut, so +# installing is one drag (README: Installation -> Mac). +# +# Usage: make_dmg.sh +set -euo pipefail + +app="${1:?usage: make_dmg.sh }" +dmg="${2:?usage: make_dmg.sh }" + +staging="$(mktemp -d)" +trap 'rm -rf "$staging"' EXIT + +# ditto keeps the bundle's symlinks, permissions and signature intact. +ditto "$app" "$staging/$(basename "$app")" +ln -s /Applications "$staging/Applications" + +rm -f "$dmg" +hdiutil create -volname "Easy Forest HDMapping" -srcfolder "$staging" -fs HFS+ -format UDZO -ov "$dmg" +hdiutil verify "$dmg" diff --git a/apps/easy_gui/packaging/windows/make_zip.py b/apps/easy_gui/packaging/windows/make_zip.py new file mode 100755 index 00000000..f95dbac1 --- /dev/null +++ b/apps/easy_gui/packaging/windows/make_zip.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +"""Builds the Easy Forest HDMapping Windows folder and zip from a build/bin/Release directory. + +Usage: make_zip.py [--search-dir DIR ...] + +The folder holds easy_gui.exe, lidar_odometry_step_1.exe (started by easy_gui from its own directory) and every DLL +they need, found recursively with `dumpbin /dependents`. The MSVC runtime (msvcp140, vcruntime140, ...) is copied +app-locally from Visual Studio's redist directory: the build runner has it installed system-wide, a user's PC may not. +Everything else must be part of Windows (api-ms-win-*, or present in System32 and not a VC runtime DLL); a DLL that +can be found neither in the build nor in --search-dir fails the script. The zip contains the files at its top level +(README: extract into a folder, double-click easy_gui.exe). +""" + +import argparse +import glob +import os +import re +import shutil +import subprocess +import sys +import zipfile +from pathlib import Path + +EXECUTABLES = ["easy_gui.exe", "lidar_odometry_step_1.exe"] +VC_RUNTIME = re.compile(r"^(msvcp\d+.*|vcruntime\d+.*|concrt\d+|vcomp\d+|vccorlib\d+)\.dll$", re.IGNORECASE) +SYSTEM32 = Path(os.environ.get("SystemRoot", r"C:\Windows")) / "System32" + + +def vswhere(*args): + exe = Path(os.environ.get("ProgramFiles(x86)", r"C:\Program Files (x86)")) / "Microsoft Visual Studio/Installer/vswhere.exe" + return subprocess.run([str(exe), "-latest", "-products", "*", *args], check=True, capture_output=True, text=True).stdout.strip() + + +def find_tools(): + vs = Path(vswhere("-property", "installationPath")) + dumpbin = sorted(glob.glob(str(vs / "VC/Tools/MSVC/*/bin/Hostx64/x64/dumpbin.exe"))) + redist = sorted(glob.glob(str(vs / "VC/Redist/MSVC/*/x64"))) + if not dumpbin or not redist: + sys.exit(f"error: dumpbin or the VC redist directory not found under {vs}") + # Highest version wins; the redist folder holds Microsoft.VC14x.CRT (and .OpenMP) subfolders. + crt_dirs = [Path(d) for d in glob.glob(str(Path(redist[-1]) / "Microsoft.VC*.CRT"))] + crt_dirs += [Path(d) for d in glob.glob(str(Path(redist[-1]) / "Microsoft.VC*.OpenMP"))] + return Path(dumpbin[-1]), crt_dirs + + +def dependents(dumpbin, binary): + out = subprocess.run([str(dumpbin), "/nologo", "/dependents", str(binary)], check=True, capture_output=True, text=True).stdout + return [line.strip() for line in out.splitlines() if line.strip().lower().endswith(".dll") and " " not in line.strip()] + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("bin_dir", type=Path) + parser.add_argument("out_dir", type=Path) + parser.add_argument("zip", type=Path) + parser.add_argument("--search-dir", type=Path, action="append", default=[]) + args = parser.parse_args() + + dumpbin, crt_dirs = find_tools() + print(f"dumpbin: {dumpbin}\nVC runtime: {', '.join(map(str, crt_dirs))}") + + if args.out_dir.exists(): + shutil.rmtree(args.out_dir) + args.out_dir.mkdir(parents=True) + + queue = [] + for name in EXECUTABLES: + src = args.bin_dir / name + if not src.is_file(): + sys.exit(f"error: {src} not found") + shutil.copy2(src, args.out_dir / name) + queue.append(args.out_dir / name) + + search = [args.bin_dir, *args.search_dir] + copied, system, errors = set(), set(), [] + while queue: + binary = queue.pop(0) + for dll in dependents(dumpbin, binary): + key = dll.lower() + if key in copied or key in system: + continue + if key.startswith(("api-ms-win-", "ext-ms-")): + system.add(key) + continue + if VC_RUNTIME.match(dll): + source = next((d / dll for d in crt_dirs if (d / dll).is_file()), None) + else: + source = next((d / dll for d in search if (d / dll).is_file()), None) + if source is None and (SYSTEM32 / dll).is_file(): + system.add(key) + continue + if source is None: + errors.append(f"{binary.name} needs {dll}, which is neither in the build, a search dir nor Windows") + continue + shutil.copy2(source, args.out_dir / source.name) + copied.add(key) + print(f" {binary.name} -> {dll} (from {source.parent})") + queue.append(args.out_dir / source.name) + + # The C++ runtime DLLs load each other dynamically in places (e.g. msvcp140 -> msvcp140_1/_2 for some + # features), so ship the whole CRT folder rather than only what dumpbin lists. + for d in crt_dirs: + if d.name.endswith(".CRT"): + for dll in d.glob("*.dll"): + if dll.name.lower() not in copied: + shutil.copy2(dll, args.out_dir / dll.name) + copied.add(dll.name.lower()) + + if errors: + print("\nPackage check FAILED:", file=sys.stderr) + for e in errors: + print(f" {e}", file=sys.stderr) + sys.exit(1) + + print(f"\nSystem DLLs (not bundled): {', '.join(sorted(system))}") + print(f"Folder contents:") + for f in sorted(args.out_dir.iterdir()): + print(f" {f.name:40s} {f.stat().st_size / 1e6:8.2f} MB") + + args.zip.unlink(missing_ok=True) + with zipfile.ZipFile(args.zip, "w", zipfile.ZIP_DEFLATED, compresslevel=9) as z: + for f in sorted(args.out_dir.iterdir()): + z.write(f, f.name) + print(f"\n{args.zip}: {args.zip.stat().st_size / 1e6:.1f} MB") + + +if __name__ == "__main__": + main() diff --git a/apps/easy_gui/src/child_process.cpp b/apps/easy_gui/src/child_process.cpp index c9c6fba3..f7efbe53 100644 --- a/apps/easy_gui/src/child_process.cpp +++ b/apps/easy_gui/src/child_process.cpp @@ -1,6 +1,7 @@ #include "child_process.h" #include +#include #include #ifdef _WIN32 @@ -240,6 +241,15 @@ namespace easy_gui return fs::path(buffer).parent_path(); } + void attachToParentConsole() + { + if (!AttachConsole(ATTACH_PARENT_PROCESS)) + return; + FILE* stream = nullptr; + freopen_s(&stream, "CONOUT$", "w", stdout); + freopen_s(&stream, "CONOUT$", "w", stderr); + } + #else // POSIX struct ChildProcess::Impl @@ -396,5 +406,9 @@ namespace easy_gui return ec ? fs::path() : exe.parent_path(); #endif } + + void attachToParentConsole() + { + } #endif } // namespace easy_gui diff --git a/apps/easy_gui/src/child_process.h b/apps/easy_gui/src/child_process.h index 802f8c86..6340c2e9 100644 --- a/apps/easy_gui/src/child_process.h +++ b/apps/easy_gui/src/child_process.h @@ -55,4 +55,9 @@ namespace easy_gui //! Directory of the running executable (empty if it cannot be determined). std::filesystem::path currentExecutableDirectory(); + + //! Windows: easy_gui.exe is a windowed program, so it has no console and printf goes nowhere. The command-line + //! modes (--headless, --export-only) call this to print into the console they were started from, if any. + //! No-op elsewhere. + void attachToParentConsole(); } // namespace easy_gui diff --git a/apps/easy_gui/src/easy_gui.manifest b/apps/easy_gui/src/easy_gui.manifest new file mode 100644 index 00000000..21db78cd --- /dev/null +++ b/apps/easy_gui/src/easy_gui.manifest @@ -0,0 +1,11 @@ + + + + + + UTF-8 + + + diff --git a/apps/easy_gui/src/easy_gui_main.cpp b/apps/easy_gui/src/easy_gui_main.cpp index 248cdc24..a05a82e9 100644 --- a/apps/easy_gui/src/easy_gui_main.cpp +++ b/apps/easy_gui/src/easy_gui_main.cpp @@ -1,6 +1,7 @@ // HDMapping Easy GUI: raw Mandeye recording -> lidar odometry (precision forestry preset) -> point cloud + trajectory. // See apps/easy_gui/docs/DESIGN.md for the decisions behind the pipeline. +#include "child_process.h" #include "export.h" #include "pipeline.h" #include "preflight.h" @@ -539,6 +540,7 @@ int main(int argc, char* argv[]) std::vector args(argv + 1, argv + argc); if (!args.empty() && (args[0] == "--headless" || args[0] == "--export-only")) { + attachToParentConsole(); const double check_band = takeCheckBand(args); Settings settings; settings.place_by_raw_time = takeFlag(args, "--place-by-raw-time"); diff --git a/apps/easy_gui/tests/test_child_process.cpp b/apps/easy_gui/tests/test_child_process.cpp index c5668532..6f32fcee 100644 --- a/apps/easy_gui/tests/test_child_process.cpp +++ b/apps/easy_gui/tests/test_child_process.cpp @@ -3,12 +3,46 @@ #include "child_process.h" #include +#include #include #include using easy_gui::ChildProcess; -#ifndef _WIN32 // uses /bin/sh; the Windows implementation is exercised by running Easy GUI +namespace +{ + std::vector readAll(ChildProcess& child) + { + std::vector lines; + std::string line; + while (child.readLine(line)) + lines.push_back(line); + return lines; + } +} // namespace + +TEST_CASE("arguments reach the child unchanged (spaces, quotes, backslashes, empty)") +{ + // easy_gui_tests itself is the child (--echo-args, see test_main.cpp). On Windows this checks the command line + // quoting, which output folders like "Documents\Easy Forest HDMapping\..." depend on. +#ifdef _WIN32 + const auto self = easy_gui::currentExecutableDirectory() / "easy_gui_tests.exe"; +#else + const auto self = easy_gui::currentExecutableDirectory() / "easy_gui_tests"; +#endif + const std::vector args = { "--echo-args", "plain", "with space", "", "quote\"inside", + "C:\\dir with space\\", "trailing\\", "a\\\\\"b" }; + ChildProcess child; + std::string error; + REQUIRE_MESSAGE(child.start(self, args, error), error); + const auto lines = readAll(child); + CHECK(child.wait() == 0); + REQUIRE(lines.size() == args.size() - 1); + for (size_t i = 1; i < args.size(); i++) + CHECK(lines[i - 1] == "[" + args[i] + "]"); +} + +#ifndef _WIN32 // uses /bin/sh TEST_CASE("stdout and stderr are merged line by line and the exit code is returned") { @@ -73,4 +107,72 @@ TEST_CASE("starting a missing program fails with a message") CHECK_FALSE(error.empty()); } +#else // cmd.exe + +namespace +{ + std::filesystem::path cmdExe() + { + const char* comspec = std::getenv("ComSpec"); + return (comspec && *comspec) ? std::filesystem::path(comspec) : std::filesystem::path("C:\\Windows\\System32\\cmd.exe"); + } +} // namespace + +TEST_CASE("stdout and stderr are merged line by line and the exit code is returned") +{ + ChildProcess child; + std::string error; + REQUIRE(child.start(cmdExe(), { "/c", "echo one& echo two 1>&2& exit /b 3" }, error)); + const auto lines = readAll(child); + CHECK(child.wait() == 3); + REQUIRE(lines.size() == 2); + CHECK(lines[0] == "one"); // '\r' of the CRLF removed + CHECK(lines[1] == "two"); +} + +TEST_CASE("stdin is closed, so a program waiting for input does not hang") +{ + ChildProcess child; + std::string error; + REQUIRE(child.start(cmdExe(), { "/c", "set /p x=& echo got-eof" }, error)); + const auto lines = readAll(child); + REQUIRE(!lines.empty()); + CHECK(lines.back() == "got-eof"); + CHECK(child.wait() != -1); +} + +TEST_CASE("kill stops the child and everything it started promptly") +{ + ChildProcess child; + std::string error; + // ping is a grandchild holding the pipe's write end (stderr): only the job object takes it down too. + REQUIRE(child.start(cmdExe(), { "/c", "echo started& ping -n 30 127.0.0.1 >nul" }, error)); + std::string line; + REQUIRE(child.readLine(line)); + CHECK(line == "started"); + + const auto t0 = std::chrono::steady_clock::now(); + std::thread killer( + [&] + { + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + child.kill(); + }); + while (child.readLine(line)) + { + } + killer.join(); + CHECK(child.wait() == -1); + CHECK(child.wasKilled()); + CHECK(std::chrono::steady_clock::now() - t0 < std::chrono::seconds(5)); +} + +TEST_CASE("starting a missing program fails with a message") +{ + ChildProcess child; + std::string error; + CHECK_FALSE(child.start("C:\\nonexistent\\program.exe", {}, error)); + CHECK_FALSE(error.empty()); +} + #endif diff --git a/apps/easy_gui/tests/test_main.cpp b/apps/easy_gui/tests/test_main.cpp index b8e3a4bf..7207bc37 100644 --- a/apps/easy_gui/tests/test_main.cpp +++ b/apps/easy_gui/tests/test_main.cpp @@ -1,2 +1,20 @@ -#define DOCTEST_CONFIG_IMPLEMENT_WITH_MAIN +#define DOCTEST_CONFIG_IMPLEMENT #include + +#include +#include + +// `easy_gui_tests --echo-args ` prints each argument as "[arg]" on its own line and exits: a child program +// that behaves the same on every platform, for the ChildProcess argument quoting test. +int main(int argc, char** argv) +{ + if (argc > 1 && std::strcmp(argv[1], "--echo-args") == 0) + { + for (int i = 2; i < argc; i++) + std::printf("[%s]\n", argv[i]); + return 0; + } + doctest::Context context; + context.applyCommandLine(argc, argv); + return context.run(); +} diff --git a/cmake/dependencies.cmake b/cmake/dependencies.cmake index 7a81177c..a8379055 100644 --- a/cmake/dependencies.cmake +++ b/cmake/dependencies.cmake @@ -147,9 +147,13 @@ message(STATUS "Using bundled Xerces-C + libE57Format (static) for E57 support") # ============================================================================ # OpenCV - Computer vision library -include(${THIRDPARTY_DIRECTORY_BINARY}/OpenCV/CMakeLists.txt) -find_package(OpenCV REQUIRED) -message(STATUS "OpenCV include dir: ${OpenCV_INCLUDE_DIRS}, OpenCV libs: ${OpenCV_LIBS}") +# Only the camera/raw-viewer apps link it, and HDMAPPING_EASY_GUI_ONLY never configures those, so that build skips +# it: no Homebrew OpenCV on macOS (built for the runner's macOS version), no ~200 MB download on Windows. +if(NOT HDMAPPING_EASY_GUI_ONLY) + include(${THIRDPARTY_DIRECTORY_BINARY}/OpenCV/CMakeLists.txt) + find_package(OpenCV REQUIRED) + message(STATUS "OpenCV include dir: ${OpenCV_INCLUDE_DIRS}, OpenCV libs: ${OpenCV_LIBS}") +endif() # PROJ - Cartographic projections library include(${THIRDPARTY_DIRECTORY_BINARY}/Proj/CMakeLists.txt) From 27882914c91954e44305ee9fda5b329c3506f442 Mon Sep 17 00:00:00 2001 From: Martin Mokros Date: Sun, 18 Oct 2026 12:49:29 +0100 Subject: [PATCH 12/23] Easy Forest Windows workflow: default Visual Studio generator, latest CMake Co-Authored-By: Claude Opus 5.5 --- .github/workflows/easy-forest-windows.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/easy-forest-windows.yml b/.github/workflows/easy-forest-windows.yml index 8c912673..6f5d92df 100644 --- a/.github/workflows/easy-forest-windows.yml +++ b/.github/workflows/easy-forest-windows.yml @@ -33,15 +33,14 @@ jobs: echo "VERSION=$(git describe --tags --always)" >> "$GITHUB_ENV" fi + # Latest CMake: it has to know the generator of whatever Visual Studio the runner image ships. - uses: lukka/get-cmake@latest - with: - cmakeVersion: "~4.0.0" # HD_CPU_OPTIMIZATION=GENERIC: AUTO adds /arch:AVX2 when the runner's CPU has it, and the release must also run # on PCs without AVX2. - name: Configure run: | - cmake -B build -G "Visual Studio 17 2022" -A x64 \ + cmake -B build -A x64 \ -DHDMAPPING_EASY_GUI_ONLY=ON \ -DBUILD_TESTING=ON \ -DHD_CPU_OPTIMIZATION=GENERIC \ From 095a4fa24f469f72edc79ae0180a0d8bb979af6f Mon Sep 17 00:00:00 2001 From: Martin Mokros Date: Sun, 18 Oct 2026 13:05:12 +0100 Subject: [PATCH 13/23] easy_gui_tests: fix cmd.exe stderr test (no space before the redirection) Co-Authored-By: Claude Opus 5.5 --- apps/easy_gui/tests/test_child_process.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/easy_gui/tests/test_child_process.cpp b/apps/easy_gui/tests/test_child_process.cpp index 6f32fcee..4b79f50f 100644 --- a/apps/easy_gui/tests/test_child_process.cpp +++ b/apps/easy_gui/tests/test_child_process.cpp @@ -122,7 +122,8 @@ TEST_CASE("stdout and stderr are merged line by line and the exit code is return { ChildProcess child; std::string error; - REQUIRE(child.start(cmdExe(), { "/c", "echo one& echo two 1>&2& exit /b 3" }, error)); + // Redirection first: in "echo two 1>&2" cmd would print the space before the redirection too. + REQUIRE(child.start(cmdExe(), { "/c", "echo one& 1>&2 echo two& exit /b 3" }, error)); const auto lines = readAll(child); CHECK(child.wait() == 3); REQUIRE(lines.size() == 2); From 2d6c3314e7afdb62af49d2add397dc1ed2d7f63b Mon Sep 17 00:00:00 2001 From: Martin Mokros Date: Sun, 18 Oct 2026 13:20:07 +0100 Subject: [PATCH 14/23] Easy Forest DESIGN.md: Windows build results Co-Authored-By: Claude Opus 5.5 --- apps/easy_gui/docs/DESIGN.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/apps/easy_gui/docs/DESIGN.md b/apps/easy_gui/docs/DESIGN.md index 847f501b..35286013 100644 --- a/apps/easy_gui/docs/DESIGN.md +++ b/apps/easy_gui/docs/DESIGN.md @@ -64,7 +64,7 @@ Code in `apps/easy_gui`: | `easy_gui_core` (static library, no raylib) | Everything except the window, so the doctest suite can link it. | | `src/preflight.*` | Classifies files exactly like step 1's `load_data` does and checks them (see ยง3). | | `src/staging.*` | Builds the staging folder of symlinks (see ยง5.6). | -| `src/child_process.*` | Runs step 1: POSIX `posix_spawn` with its own process group; Windows `CreateProcess` with a job object (written, not yet compiled on Windows). | +| `src/child_process.*` | Runs step 1: POSIX `posix_spawn` with its own process group; Windows `CreateProcess` with a job object (built and tested on the Windows CI runner). | | `src/progress_parser.*` | Turns step 1's log lines into phases and a progress fraction. | | `src/forestry_preset.*` | Writes the step 1 parameter file with step 1's own `TomlIO`. | | `src/export.*`, `src/las14_writer.*` | The export (ยง6) and a streaming LAS 1.4 point format 6 LAZ writer with a hand-written extra-bytes VLR. | @@ -411,13 +411,15 @@ always reports the breakdown, warns below 1.5ร— the estimate and refuses below 1 `liblaszip`, `libtbb`, `libglut` in `Contents/Frameworks`, rpaths rewritten), re-signs it ad hoc and fails unless every bundled binary has minos <= 13.0 and references nothing outside the bundle, `/usr/lib` and `/System`. The build also fails on the linker's "built for newer macOS version" warning, which a static library would only show - there. `make_dmg.sh` adds the Applications shortcut. Local run: app 7.9 MB, DMG 3.5 MB; the bundled app processed - `mini4` headless. + there. `make_dmg.sh` adds the Applications shortcut. Local and CI run: app 7.9 MB, DMG 3.6 MB; locally the bundled app + processed `mini4` headless. - Windows (`windows-latest`, x64): `easy_gui.exe` is a WIN32-subsystem program (`/ENTRY:mainCRTStartup`, so `main` stays), `--headless`/`--export-only` attach to the parent console. A manifest sets UTF-8 as the process code page, because the file dialog, drag and drop and `ChildProcess` all use UTF-8. `HD_CPU_OPTIMIZATION=GENERIC`: `AUTO` adds `/arch:AVX2` from the runner's CPU. `make_zip.py` collects the DLLs with `dumpbin /dependents`, copies the MSVC runtime app-locally from VS's redist folder and fails on any DLL that is neither bundled nor part of Windows. + First CI run: folder 7.1 MB (two exes, `laszip3`, `freeglut`, `tbb12` and the MSVC runtime DLLs), zip 3.3 MB. + PROJ's DLL is not needed: nothing Easy GUI calls references it, so the linker drops the import. - `easy_gui_tests` cover `ChildProcess` on Windows too (cmd.exe; argument quoting through `--echo-args`). - No app icon yet (none exists for Easy GUI); no code signing or notarization (README explains the warnings). From 8cc5a0335fe276eed34ecf001c493a165149501c Mon Sep 17 00:00:00 2001 From: Martin Mokros Date: Sun, 18 Oct 2026 14:14:59 +0100 Subject: [PATCH 15/23] Easy Forest HDMapping: own version 0.1.0 next to HDMapping's Window title, README.txt and easy_gui.log show "Easy Forest HDMapping 0.1.0 (HDMapping 0.105.0)". The LAZ header's 32-character fields get one name each: generating software "Easy Forest HDMapping 0.1.0", system identifier "HDMapping 0.105.0" (Las14Writer::open takes the system identifier now). Co-Authored-By: Claude Opus 5.5 --- apps/easy_gui/CMakeLists.txt | 7 ++++++- apps/easy_gui/docs/DESIGN.md | 5 +++++ apps/easy_gui/src/easy_gui_main.cpp | 4 ++-- apps/easy_gui/src/export.cpp | 6 +++++- apps/easy_gui/src/las14_writer.cpp | 3 ++- apps/easy_gui/src/las14_writer.h | 1 + apps/easy_gui/src/pipeline.cpp | 5 ++--- apps/easy_gui/src/version.h | 18 ++++++++++++++++++ apps/easy_gui/tests/test_export.cpp | 1 + 9 files changed, 42 insertions(+), 8 deletions(-) create mode 100644 apps/easy_gui/src/version.h diff --git a/apps/easy_gui/CMakeLists.txt b/apps/easy_gui/CMakeLists.txt index ffe412bc..b069dda3 100644 --- a/apps/easy_gui/CMakeLists.txt +++ b/apps/easy_gui/CMakeLists.txt @@ -2,6 +2,10 @@ cmake_minimum_required(VERSION 4.0.0) project(easy_gui) +# Easy Forest HDMapping's own version: shown next to HDMapping's in the window title, README.txt, easy_gui.log and +# the LAZ header. Keep it equal to the easy-forest-v* release tag. +set(EASY_FOREST_VERSION "0.1.0") + # easy_gui_core -- everything except the window: pre-flight check, staging, the # step 1 child process and its progress parsing, and the pipeline that ties # them together. A separate static library so the doctest suite in tests/ can @@ -30,12 +34,13 @@ add_library(easy_gui_core STATIC src/progress_parser.cpp src/progress_parser.h src/settings.h src/staging.cpp src/staging.h + src/version.h ../lidar_odometry_step_1/lidar_odometry_utils.cpp ../lidar_odometry_step_1/toml_io.cpp ) target_compile_definitions(easy_gui_core - PUBLIC WITH_GUI=0 + PUBLIC WITH_GUI=0 EASY_FOREST_VERSION="${EASY_FOREST_VERSION}" PRIVATE HDMAPPING_BINARY_PREFIX="${HDMAPPING_BINARY_PREFIX}" UTL_PROFILER_DISABLE) target_include_directories(easy_gui_core diff --git a/apps/easy_gui/docs/DESIGN.md b/apps/easy_gui/docs/DESIGN.md index 35286013..6314ab17 100644 --- a/apps/easy_gui/docs/DESIGN.md +++ b/apps/easy_gui/docs/DESIGN.md @@ -6,6 +6,11 @@ collaborators and reviewers who know HDMapping. The user guide is [../README.md] Naming: the user-facing name is **Easy Forest HDMapping** (window title, user guide, release packages). Internally, the code keeps the name Easy GUI (`apps/easy_gui`, the `easy_gui` binary and targets), and so does this document. +Version: Easy Forest HDMapping has its own version (`EASY_FOREST_VERSION` in `apps/easy_gui/CMakeLists.txt`, +equal to the `easy-forest-v*` tag). Window title, README.txt and easy_gui.log show both, e.g. "Easy Forest HDMapping +0.1.0 (HDMapping 0.105.0)". The LAZ header's 32-character fields cannot hold that: generating software is +"Easy Forest HDMapping 0.1.0", system identifier "HDMapping 0.105.0". + Contents: 1. [Goal and scope](#1-goal-and-scope) diff --git a/apps/easy_gui/src/easy_gui_main.cpp b/apps/easy_gui/src/easy_gui_main.cpp index a05a82e9..df701327 100644 --- a/apps/easy_gui/src/easy_gui_main.cpp +++ b/apps/easy_gui/src/easy_gui_main.cpp @@ -6,9 +6,9 @@ #include "pipeline.h" #include "preflight.h" #include "settings.h" +#include "version.h" #include -#include #include #include @@ -574,7 +574,7 @@ int main(int argc, char* argv[]) app.setRecording(argv[first_positional]); SetConfigFlags(FLAG_WINDOW_RESIZABLE | FLAG_MSAA_4X_HINT); - InitWindow(900, 780, ("Easy Forest HDMapping " + std::string(HDMAPPING_CONFIGURE_PROJECT_VERSION)).c_str()); + InitWindow(900, 780, versionString().c_str()); raylib_widgets::fitWindowToScreen(); // raylib's default exit key (Esc) would close the window mid-run. SetExitKey(KEY_NULL); diff --git a/apps/easy_gui/src/export.cpp b/apps/easy_gui/src/export.cpp index a9777d3f..df0a25c1 100644 --- a/apps/easy_gui/src/export.cpp +++ b/apps/easy_gui/src/export.cpp @@ -5,8 +5,10 @@ #include #include +#include "version.h" #include #include + #include #include @@ -212,7 +214,7 @@ namespace easy_gui out << "Easy Forest HDMapping results\n" << "==========================\n\n" - << "Made by Easy Forest HDMapping " << HDMAPPING_CONFIGURE_PROJECT_VERSION << " from the recording\n" + << "Made by " << versionString() << " from the recording\n" << " " << request.recording.laz_files.front().parent_path().string() << "\n" << "with lidar odometry (precision forestry settings). No loop closure and no georeferencing: coordinates\n" << "are in metres in a local frame that starts at the scanner's first position (z up, from the IMU).\n\n" @@ -421,7 +423,9 @@ namespace easy_gui kAttributes, offset, kCoordinateScale, + // Both header fields hold 32 characters, too few for versionString(): one name each. std::string("HDMapping ") + HDMAPPING_CONFIGURE_PROJECT_VERSION, + std::string("Easy Forest HDMapping ") + kEasyForestVersion, error)) { error = "Could not create the point cloud file '" + request.laz_file.string() + "' (" + error + ")."; diff --git a/apps/easy_gui/src/las14_writer.cpp b/apps/easy_gui/src/las14_writer.cpp index a8dc325a..90f5c0cf 100644 --- a/apps/easy_gui/src/las14_writer.cpp +++ b/apps/easy_gui/src/las14_writer.cpp @@ -64,6 +64,7 @@ namespace easy_gui const std::vector& attributes, const Eigen::Vector3d& offset, double scale, + const std::string& system_identifier, const std::string& generating_software, std::string& error) { @@ -89,7 +90,7 @@ namespace easy_gui header->global_encoding = (1 << 4); // bit 0 = 0: not adjusted GPS time; bit 4: WKT (required for PF6) header->version_major = 1; header->version_minor = 4; - copyField(header->system_identifier, sizeof(header->system_identifier), "Easy Forest HDMapping"); + copyField(header->system_identifier, sizeof(header->system_identifier), system_identifier); copyField(header->generating_software, sizeof(header->generating_software), generating_software); { const std::time_t now = std::time(nullptr); diff --git a/apps/easy_gui/src/las14_writer.h b/apps/easy_gui/src/las14_writer.h index a1af6ada..15646486 100644 --- a/apps/easy_gui/src/las14_writer.h +++ b/apps/easy_gui/src/las14_writer.h @@ -50,6 +50,7 @@ namespace easy_gui const std::vector& attributes, const Eigen::Vector3d& offset, double scale, + const std::string& system_identifier, const std::string& generating_software, std::string& error); bool write(const Las14Point& point, std::string& error); diff --git a/apps/easy_gui/src/pipeline.cpp b/apps/easy_gui/src/pipeline.cpp index 6b070c8b..1fed7b45 100644 --- a/apps/easy_gui/src/pipeline.cpp +++ b/apps/easy_gui/src/pipeline.cpp @@ -2,8 +2,7 @@ #include "forestry_preset.h" #include "staging.h" - -#include +#include "version.h" #include #include @@ -209,7 +208,7 @@ namespace easy_gui } log_file_.open(out.log_file, std::ios::out | std::ios::app); - log("[easy_gui] Easy Forest HDMapping " + std::string(HDMAPPING_CONFIGURE_PROJECT_VERSION) + ", started " + timestamp()); + log("[easy_gui] " + versionString() + ", started " + timestamp()); log("[easy_gui] recording: " + request_.recording.string()); log("[easy_gui] output: " + out.root.string()); log("[easy_gui] step 1 program: " + request_.step1_executable.string()); diff --git a/apps/easy_gui/src/version.h b/apps/easy_gui/src/version.h new file mode 100644 index 00000000..b4ef5bfb --- /dev/null +++ b/apps/easy_gui/src/version.h @@ -0,0 +1,18 @@ +#pragma once + +#include + +#include + +namespace easy_gui +{ + //! Easy Forest HDMapping's own release version (EASY_FOREST_VERSION in apps/easy_gui/CMakeLists.txt, matching the + //! easy-forest-v* release tags). HDMapping's version is a different number, so user-facing text names both. + inline constexpr const char* kEasyForestVersion = EASY_FOREST_VERSION; + + //! "Easy Forest HDMapping 0.1.0 (HDMapping 0.105.0)": window title, README.txt, easy_gui.log. + inline std::string versionString() + { + return std::string("Easy Forest HDMapping ") + kEasyForestVersion + " (HDMapping " + HDMAPPING_CONFIGURE_PROJECT_VERSION + ")"; + } +} // namespace easy_gui diff --git a/apps/easy_gui/tests/test_export.cpp b/apps/easy_gui/tests/test_export.cpp index 01eaf809..b4e250a5 100644 --- a/apps/easy_gui/tests/test_export.cpp +++ b/apps/easy_gui/tests/test_export.cpp @@ -252,6 +252,7 @@ TEST_CASE("Las14Writer writes LAS 1.4 point format 6 with named, unscaled extra { { ExtraBytesAttribute::Type::F32, "range_m", "range" }, { ExtraBytesAttribute::Type::U8, "point_origin", "flag" } }, Eigen::Vector3d(100.0, 200.0, 0.0), 0.0001, + "test system", "test", error)); #pragma pack(push, 1) From 1bb7df500c0d2fee612398121d67d4f73ed5ede3 Mon Sep 17 00:00:00 2001 From: Martin Mokros Date: Sun, 18 Oct 2026 14:15:19 +0100 Subject: [PATCH 16/23] Easy Forest README: download from the releases page; Windows status Both download placeholders point to the releases page (not /releases/latest, which skips pre-releases). Windows: built and unit-tested, not yet tested on a real PC. Co-Authored-By: Claude Opus 5.5 --- apps/easy_gui/README.md | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/apps/easy_gui/README.md b/apps/easy_gui/README.md index 3312a92e..81a24dd7 100644 --- a/apps/easy_gui/README.md +++ b/apps/easy_gui/README.md @@ -38,7 +38,7 @@ can continue there. | | Mac | Windows | |---|---|---| -| System | macOS 13 (Ventura) or newer | Coming soon, not yet tested | +| System | macOS 13 (Ventura) or newer | Windows 10 or 11, 64-bit (not yet tested on a real PC) | | Processor | Apple silicon (M1 or newer). Intel Macs are not supported in v1. | 64-bit Intel or AMD processor | | Memory (RAM) | 16 GB recommended | 16 GB recommended | @@ -61,7 +61,8 @@ need to do this once. ### Mac -1. Download **`EasyForestHDMapping-mac.dmg`** from *[download link โ€” coming with the first release]*. +1. Open the [releases page](https://github.com/MartinMokros/HDMapping-Easy-GUI/releases) and download the newest Easy Forest HDMapping release: the file + **`EasyForestHDMapping-mac.dmg`** under *Assets*. 2. Double-click the `.dmg` file. In the window that opens, drag **Easy Forest HDMapping** onto the **Applications** folder. 3. Open your **Applications** folder and double-click **Easy Forest HDMapping**. @@ -74,11 +75,13 @@ need to do this once. From now on Easy Forest HDMapping opens normally. If macOS ever says the app **"is damaged and can't be opened"**, see [Troubleshooting](#troubleshooting). -### Windows (coming soon, not yet tested) +### Windows (not yet tested on a real PC) -The Windows version has not been built and tested yet. These steps are what installation will look like: +The Windows version is built and its automatic tests pass, but nobody has used it on a real Windows PC yet. If +something goes wrong, please report it (see [Getting help](#getting-help)). -1. Download **`EasyForestHDMapping-windows.zip`** from *[download link โ€” coming with the first release]*. +1. Open the [releases page](https://github.com/MartinMokros/HDMapping-Easy-GUI/releases) and download the newest Easy Forest HDMapping release: the file + **`EasyForestHDMapping-windows.zip`** under *Assets*. 2. Before unpacking: right-click the downloaded `.zip` file โ†’ **Properties** โ†’ on the **General** tab, tick **Unblock** at the bottom โ†’ **OK**. (If there is no Unblock box, skip this step.) 3. Right-click the `.zip` file โ†’ **Extract Allโ€ฆ** โ†’ choose a folder, for example `Documents\Easy Forest HDMapping` From 8e332c048e65c5c99a0bbb2cf69914d6aed278a2 Mon Sep 17 00:00:00 2001 From: Martin Mokros Date: Sun, 18 Oct 2026 14:15:47 +0100 Subject: [PATCH 17/23] Easy Forest release workflows: 0.x releases are pre-releases; tag must match EASY_FOREST_VERSION Co-Authored-By: Claude Opus 5.5 --- .github/workflows/easy-forest-macos.yml | 12 +++++++++++- .github/workflows/easy-forest-windows.yml | 12 +++++++++++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/.github/workflows/easy-forest-macos.yml b/.github/workflows/easy-forest-macos.yml index 77f4ede1..e2b1b3b0 100644 --- a/.github/workflows/easy-forest-macos.yml +++ b/.github/workflows/easy-forest-macos.yml @@ -31,6 +31,12 @@ jobs: run: | if [[ "$GITHUB_REF" == refs/tags/easy-forest-v* ]]; then echo "VERSION=${GITHUB_REF_NAME#easy-forest-v}" >> "$GITHUB_ENV" + # The app shows EASY_FOREST_VERSION from CMake; it must be the tag's version. + cmake_version=$(sed -n 's/^set(EASY_FOREST_VERSION "\(.*\)")$/\1/p' apps/easy_gui/CMakeLists.txt) + if [[ "$cmake_version" != "${GITHUB_REF_NAME#easy-forest-v}" ]]; then + echo "::error::Tag $GITHUB_REF_NAME does not match EASY_FOREST_VERSION $cmake_version in apps/easy_gui/CMakeLists.txt" + exit 1 + fi else echo "VERSION=$(git describe --tags --always)" >> "$GITHUB_ENV" fi @@ -115,13 +121,17 @@ jobs: retention-days: 90 # The macOS and Windows workflows both run on the same tag: whichever gets here first creates the release. + # 0.x versions and versions with a suffix (0.2.0-rc1) are pre-releases. Created with GITHUB_TOKEN, so the + # release does not trigger other workflows (upstream windows.yml runs on release: published). - name: Attach to GitHub release if: startsWith(github.ref, 'refs/tags/easy-forest-v') env: GH_TOKEN: ${{ github.token }} run: | + PRERELEASE="" + if [[ "$VERSION" == 0.* || "$VERSION" == *-* ]]; then PRERELEASE="--prerelease"; fi gh release view "$GITHUB_REF_NAME" > /dev/null 2>&1 \ - || gh release create "$GITHUB_REF_NAME" --verify-tag --title "Easy Forest HDMapping $VERSION" \ + || gh release create "$GITHUB_REF_NAME" --verify-tag --title "Easy Forest HDMapping $VERSION" $PRERELEASE \ --notes "Installation and use: [apps/easy_gui/README.md](https://github.com/${GITHUB_REPOSITORY}/blob/${GITHUB_REF_NAME}/apps/easy_gui/README.md)" \ || gh release view "$GITHUB_REF_NAME" > /dev/null gh release upload "$GITHUB_REF_NAME" dist/EasyForestHDMapping-mac.dmg --clobber diff --git a/.github/workflows/easy-forest-windows.yml b/.github/workflows/easy-forest-windows.yml index 6f5d92df..a99032b2 100644 --- a/.github/workflows/easy-forest-windows.yml +++ b/.github/workflows/easy-forest-windows.yml @@ -29,6 +29,12 @@ jobs: run: | if [[ "$GITHUB_REF" == refs/tags/easy-forest-v* ]]; then echo "VERSION=${GITHUB_REF_NAME#easy-forest-v}" >> "$GITHUB_ENV" + # The app shows EASY_FOREST_VERSION from CMake; it must be the tag's version. + cmake_version=$(sed -n 's/^set(EASY_FOREST_VERSION "\(.*\)")$/\1/p' apps/easy_gui/CMakeLists.txt) + if [[ "$cmake_version" != "${GITHUB_REF_NAME#easy-forest-v}" ]]; then + echo "::error::Tag $GITHUB_REF_NAME does not match EASY_FOREST_VERSION $cmake_version in apps/easy_gui/CMakeLists.txt" + exit 1 + fi else echo "VERSION=$(git describe --tags --always)" >> "$GITHUB_ENV" fi @@ -89,13 +95,17 @@ jobs: retention-days: 90 # The macOS and Windows workflows both run on the same tag: whichever gets here first creates the release. + # 0.x versions and versions with a suffix (0.2.0-rc1) are pre-releases. Created with GITHUB_TOKEN, so the + # release does not trigger other workflows (upstream windows.yml runs on release: published). - name: Attach to GitHub release if: startsWith(github.ref, 'refs/tags/easy-forest-v') env: GH_TOKEN: ${{ github.token }} run: | + PRERELEASE="" + if [[ "$VERSION" == 0.* || "$VERSION" == *-* ]]; then PRERELEASE="--prerelease"; fi gh release view "$GITHUB_REF_NAME" > /dev/null 2>&1 \ - || gh release create "$GITHUB_REF_NAME" --verify-tag --title "Easy Forest HDMapping $VERSION" \ + || gh release create "$GITHUB_REF_NAME" --verify-tag --title "Easy Forest HDMapping $VERSION" $PRERELEASE \ --notes "Installation and use: [apps/easy_gui/README.md](https://github.com/${GITHUB_REPOSITORY}/blob/${GITHUB_REF_NAME}/apps/easy_gui/README.md)" \ || gh release view "$GITHUB_REF_NAME" > /dev/null gh release upload "$GITHUB_REF_NAME" dist/EasyForestHDMapping-windows.zip --clobber From 6ba8600066b90ce58a5d802fb6c414a254a3cb65 Mon Sep 17 00:00:00 2001 From: Martin Mokros Date: Sun, 18 Oct 2026 14:53:48 +0100 Subject: [PATCH 18/23] easy_gui: resume into an output folder that holds only an unfinished earlier attempt Groundwork for batch mode: runPreflight(..., resume_unfinished) accepts an output folder holding nothing but Easy GUI's own run files, and the pipeline clears it (keeping and continuing easy_gui.log) when PipelineRequest::resume_unfinished is set. Single mode is unchanged. OutputLayout gains the batch done marker path; availableSpace() and formatBytes() are exported for the batch disk check. Co-Authored-By: Claude Opus 5.5 --- apps/easy_gui/src/pipeline.cpp | 16 +++++- apps/easy_gui/src/pipeline.h | 4 ++ apps/easy_gui/src/preflight.cpp | 72 +++++++++++++++++++------- apps/easy_gui/src/preflight.h | 17 +++++- apps/easy_gui/tests/test_preflight.cpp | 48 +++++++++++++++++ 5 files changed, 135 insertions(+), 22 deletions(-) diff --git a/apps/easy_gui/src/pipeline.cpp b/apps/easy_gui/src/pipeline.cpp index 1fed7b45..8444ee9a 100644 --- a/apps/easy_gui/src/pipeline.cpp +++ b/apps/easy_gui/src/pipeline.cpp @@ -48,6 +48,7 @@ namespace easy_gui , point_cloud(root_ / "point_cloud.laz") , trajectory(root_ / "trajectory.csv") , readme(root_ / "README.txt") + , done_marker(root_ / "easy_gui_done.txt") { } @@ -185,7 +186,8 @@ namespace easy_gui // --- 1. re-check: the recording or the disk may have changed since the user pressed "Check" setStage(PipelineStage::Checking); - const PreflightResult check = runPreflight(request_.recording, request_.output_folder); + const PreflightResult check = runPreflight(request_.recording, request_.output_folder, request_.resume_unfinished); + const bool replace_unfinished = request_.resume_unfinished && holdsUnfinishedOutput(out.root); if (!check.ok()) { std::string first_error; @@ -220,6 +222,18 @@ namespace easy_gui } for (const auto& f : check.findings) log(std::string("[easy_gui] check ") + severityTag(f.severity) + ": " + f.message); + if (replace_unfinished) + { + // Only Easy GUI's own files are here (holdsUnfinishedOutput); a half-written result must not survive into + // this run, and step 1 needs a fresh staging folder. + std::vector leftovers; + for (const auto& entry : fs::directory_iterator(out.root, ec)) + if (entry.path().filename() != out.log_file.filename()) + leftovers.push_back(entry.path()); + for (const auto& p : leftovers) + fs::remove_all(p, ec); + log("[easy_gui] removed the files of an unfinished earlier attempt"); + } // --- 2. staging folder + parameter file setStage(PipelineStage::Staging); diff --git a/apps/easy_gui/src/pipeline.h b/apps/easy_gui/src/pipeline.h index bd38f981..1168b705 100644 --- a/apps/easy_gui/src/pipeline.h +++ b/apps/easy_gui/src/pipeline.h @@ -27,6 +27,9 @@ namespace easy_gui Settings settings; //! Dev/test only (ExportRequest::check_band_outer_m): > 0 adds alignment-check points to the export. double check_band_outer_m = 0.0; + //! Batch mode: the output folder may hold an unfinished earlier attempt (holdsUnfinishedOutput); everything in + //! it except easy_gui.log is removed before the run, and the log is continued. + bool resume_unfinished = false; }; enum class PipelineState @@ -74,6 +77,7 @@ namespace easy_gui std::filesystem::path point_cloud; // point_cloud.laz: the exported point cloud std::filesystem::path trajectory; // trajectory.csv std::filesystem::path readme; // README.txt: describes the two files above + std::filesystem::path done_marker; // easy_gui_done.txt: written by batch mode once the result is complete }; //! Locates lidar_odometry_step_1 next to the Easy GUI executable (plain build name or the installed, diff --git a/apps/easy_gui/src/preflight.cpp b/apps/easy_gui/src/preflight.cpp index 3dab021a..070004da 100644 --- a/apps/easy_gui/src/preflight.cpp +++ b/apps/easy_gui/src/preflight.cpp @@ -31,14 +31,6 @@ namespace easy_gui return s; } - std::string formatBytes(uint64_t bytes) - { - std::ostringstream out; - out.precision(1); - out << std::fixed << static_cast(bytes) / 1e9 << " GB"; - return out.str(); - } - std::string names(const std::vector& list, size_t max_shown = 5) { std::string out; @@ -236,6 +228,45 @@ namespace easy_gui return true; } + std::string formatBytes(uint64_t bytes) + { + std::ostringstream out; + out.precision(1); + out << std::fixed << static_cast(bytes) / 1e9 << " GB"; + return out.str(); + } + + bool availableSpace(const fs::path& path, uint64_t& available, fs::path& checked) + { + std::error_code ec; + checked = fs::absolute(path, ec); + while (!checked.empty() && !fs::exists(checked, ec) && checked != checked.parent_path()) + checked = checked.parent_path(); + const auto space = fs::space(checked, ec); + if (ec) + return false; + available = space.available; + return true; + } + + bool holdsUnfinishedOutput(const fs::path& folder) + { + // NOTE: the names of OutputLayout (pipeline.h), which this library level cannot include; a test keeps the two + // in sync. + static const std::set kRunEntries = { "easy_gui.log", "work", "point_cloud.laz", "trajectory.csv", "README.txt" }; + std::error_code ec; + if (!fs::is_directory(folder, ec)) + return false; + bool any = false; + for (const auto& entry : fs::directory_iterator(folder, ec)) + { + if (!kRunEntries.contains(entry.path().filename().string())) + return false; + any = true; + } + return any && !ec; + } + uint64_t estimateWorkBytes(uint64_t total_points) { return total_points * 120; @@ -280,7 +311,7 @@ namespace easy_gui return files; } - PreflightResult runPreflight(const fs::path& recording, const fs::path& output_folder) + PreflightResult runPreflight(const fs::path& recording, const fs::path& output_folder, bool resume_unfinished) { PreflightResult r; auto add = [&r](Severity s, std::string m) @@ -431,24 +462,25 @@ namespace easy_gui "(it may be on the scanner's USB stick)."); else { - if (fs::exists(output_folder, ec) && !fs::is_empty(output_folder, ec)) + if (resume_unfinished && holdsUnfinishedOutput(output_folder)) + add(Severity::Ok, + "The output folder holds an unfinished earlier attempt. It will be replaced; its log is kept and continued."); + else if (fs::exists(output_folder, ec) && !fs::is_empty(output_folder, ec)) add(Severity::Error, "The output folder '" + output_folder.string() + "' already contains files. Choose an empty or new folder."); - fs::path existing = fs::absolute(output_folder, ec); - while (!existing.empty() && !fs::exists(existing, ec) && existing != existing.parent_path()) - existing = existing.parent_path(); - const auto space = fs::space(existing, ec); - if (ec) + uint64_t available = 0; + fs::path existing; + if (!availableSpace(output_folder, available, existing)) add(Severity::Warning, "Could not determine the free disk space at '" + existing.string() + "'."); - else if (space.available < r.estimated_disk_bytes) + else if (available < r.estimated_disk_bytes) add(Severity::Error, - "Not enough free disk space for the output folder: " + need + " needed, " + formatBytes(space.available) + + "Not enough free disk space for the output folder: " + need + " needed, " + formatBytes(available) + " free. Free some space or choose a folder on another disk."); - else if (space.available < r.estimated_disk_bytes + r.estimated_disk_bytes / 2) - add(Severity::Warning, "Free disk space is tight: " + need + " needed, " + formatBytes(space.available) + " free."); + else if (available < r.estimated_disk_bytes + r.estimated_disk_bytes / 2) + add(Severity::Warning, "Free disk space is tight: " + need + " needed, " + formatBytes(available) + " free."); else - disk_summary = "Disk space: " + need + " needed, " + formatBytes(space.available) + " free."; + disk_summary = "Disk space: " + need + " needed, " + formatBytes(available) + " free."; } if (!disk_summary.empty() && r.ok()) r.findings.push_back({ Severity::Ok, disk_summary }); diff --git a/apps/easy_gui/src/preflight.h b/apps/easy_gui/src/preflight.h index 78805bb9..aea965c0 100644 --- a/apps/easy_gui/src/preflight.h +++ b/apps/easy_gui/src/preflight.h @@ -59,7 +59,22 @@ namespace easy_gui //! Checks the recording and the output folder before anything is started. Catches, with a plain message, the //! cases where step 1 would otherwise abort() (unreadable LAZ), exit() (calibration mismatch) or silently produce //! something incomplete (unpaired files, non-overlapping clocks). - PreflightResult runPreflight(const std::filesystem::path& recording, const std::filesystem::path& output_folder); + //! `resume_unfinished` (batch mode): an output folder that holds only an unfinished earlier attempt + //! (holdsUnfinishedOutput) is accepted too; the pipeline clears it before the run. + PreflightResult runPreflight( + const std::filesystem::path& recording, const std::filesystem::path& output_folder, bool resume_unfinished = false); + + //! True if `folder` exists, is not empty and holds nothing but what an Easy GUI run writes (see OutputLayout): + //! easy_gui.log, work/, point_cloud.laz, trajectory.csv, README.txt. A finished batch result also holds the done + //! marker, so it is not "unfinished". + bool holdsUnfinishedOutput(const std::filesystem::path& folder); + + //! Free bytes on the disk that holds `path`, or of its nearest existing parent (the output folder usually does + //! not exist yet). False if it cannot be determined; `checked` is the folder that was asked. + bool availableSpace(const std::filesystem::path& path, uint64_t& available, std::filesystem::path& checked); + + //! "1.2 GB" + std::string formatBytes(uint64_t bytes); //! Temporary disk space of a run: step 1's caches (original + decimated + concatenated chunk points, ~45 B/point //! each with decimation 0.01). Measured peak on a 117 M-point forest walk: ~108 B per raw point; 120 B leaves diff --git a/apps/easy_gui/tests/test_preflight.cpp b/apps/easy_gui/tests/test_preflight.cpp index cebf0c3e..0e782506 100644 --- a/apps/easy_gui/tests/test_preflight.cpp +++ b/apps/easy_gui/tests/test_preflight.cpp @@ -1,5 +1,6 @@ #include +#include "pipeline.h" #include "preflight.h" #include "test_helpers.h" @@ -135,3 +136,50 @@ TEST_CASE("isSameOrInside") CHECK_FALSE(isSameOrInside("/a/bc", "/a/b")); CHECK_FALSE(isSameOrInside("/a", "/a/b")); } + +TEST_CASE("resume_unfinished accepts an output folder that holds only an earlier attempt's own files") +{ + TempDir tmp; + easy_gui_test::writeRecording(tmp.path / "rec"); + const fs::path out = tmp.path / "out"; + fs::create_directories(out / "work" / "lio_session"); + easy_gui_test::writeText(out / "easy_gui.log", "earlier attempt\n"); + easy_gui_test::writeText(out / "point_cloud.laz", "half written"); + CHECK(holdsUnfinishedOutput(out)); + + CHECK_FALSE(runPreflight(tmp.path / "rec", out).ok()); // single mode: still refused + const auto r = runPreflight(tmp.path / "rec", out, true); + CHECK(r.ok()); + CHECK(hasFinding(r, Severity::Ok, "unfinished earlier attempt")); + + SUBCASE("a finished batch result (done marker) is not unfinished") + { + easy_gui_test::writeText(out / "easy_gui_done.txt", "done"); + CHECK_FALSE(holdsUnfinishedOutput(out)); + CHECK_FALSE(runPreflight(tmp.path / "rec", out, true).ok()); + } + SUBCASE("anything else in the folder is not ours") + { + easy_gui_test::writeText(out / "notes.txt", "mine"); + CHECK_FALSE(holdsUnfinishedOutput(out)); + CHECK_FALSE(runPreflight(tmp.path / "rec", out, true).ok()); + } + SUBCASE("missing and empty folders are not unfinished attempts") + { + CHECK_FALSE(holdsUnfinishedOutput(tmp.path / "missing")); + fs::create_directories(tmp.path / "empty"); + CHECK_FALSE(holdsUnfinishedOutput(tmp.path / "empty")); + } +} + +TEST_CASE("holdsUnfinishedOutput knows every file of OutputLayout except the done marker") +{ + TempDir tmp; + const OutputLayout out(tmp.path / "out"); + fs::create_directories(out.session); + for (const auto& file : { out.log_file, out.point_cloud, out.trajectory, out.readme, out.step1_parameters }) + easy_gui_test::writeText(file, "x"); + CHECK(holdsUnfinishedOutput(out.root)); + easy_gui_test::writeText(out.done_marker, "x"); + CHECK_FALSE(holdsUnfinishedOutput(out.root)); +} From cc7d703e39ac07c96869ca32e5bb47bc210ab2e1 Mon Sep 17 00:00:00 2001 From: Martin Mokros Date: Sun, 18 Oct 2026 14:59:57 +0100 Subject: [PATCH 19/23] easy_gui: batch core (discovery, output names, done markers, disk check, runner) New batch module in easy_gui_core, used by the GUI and --batch in the next commits: - discoverRecordings: recordings among a parent folder's direct subfolders, plus the parent itself; other folders are listed with the reason. - output names: validation (valid on Windows and macOS), case-insensitive collisions, defaults that avoid folders holding someone else's files. - done marker (easy_gui_done.txt) identifying the recording by its files, so a restarted batch skips finished recordings even if the stick is mounted elsewhere. - batch disk check: largest temporary need + all results. - BatchRunner: sequential runs through the unchanged Pipeline; a failure marks the recording failed and continues; stop after current / stop now; batch.log and batch_summary.csv (rewritten after every recording); weighted progress and ETA. - SleepBlocker: IOPMAssertion on macOS, SetThreadExecutionState on Windows. Tests use a fake RecordingJob. Co-Authored-By: Claude Opus 5.5 --- apps/easy_gui/CMakeLists.txt | 7 + apps/easy_gui/src/batch.cpp | 768 ++++++++++++++++++++++++++++ apps/easy_gui/src/batch.h | 250 +++++++++ apps/easy_gui/src/sleep_blocker.cpp | 49 ++ apps/easy_gui/src/sleep_blocker.h | 31 ++ apps/easy_gui/tests/CMakeLists.txt | 1 + apps/easy_gui/tests/test_batch.cpp | 574 +++++++++++++++++++++ 7 files changed, 1680 insertions(+) create mode 100644 apps/easy_gui/src/batch.cpp create mode 100644 apps/easy_gui/src/batch.h create mode 100644 apps/easy_gui/src/sleep_blocker.cpp create mode 100644 apps/easy_gui/src/sleep_blocker.h create mode 100644 apps/easy_gui/tests/test_batch.cpp diff --git a/apps/easy_gui/CMakeLists.txt b/apps/easy_gui/CMakeLists.txt index b069dda3..7e0f5720 100644 --- a/apps/easy_gui/CMakeLists.txt +++ b/apps/easy_gui/CMakeLists.txt @@ -25,6 +25,7 @@ set(EASY_FOREST_VERSION "0.1.0") # WITH_GUI=0 is PUBLIC because LidarOdometryParams changes layout under # WITH_GUI (clear_color), so every TU that sees it must agree. add_library(easy_gui_core STATIC + src/batch.cpp src/batch.h src/child_process.cpp src/child_process.h src/export.cpp src/export.h src/forestry_preset.cpp src/forestry_preset.h @@ -33,6 +34,7 @@ add_library(easy_gui_core STATIC src/preflight.cpp src/preflight.h src/progress_parser.cpp src/progress_parser.h src/settings.h + src/sleep_blocker.cpp src/sleep_blocker.h src/staging.cpp src/staging.h src/version.h ../lidar_odometry_step_1/lidar_odometry_utils.cpp @@ -71,6 +73,11 @@ target_link_libraries(easy_gui_core PUBLIC ${PLATFORM_LASZIP_LIB} ${PLATFORM_MISCELLANEOUS_LIBS}) +# Batch mode keeps the computer awake (src/sleep_blocker.cpp): IOKit power assertions on macOS. +if(APPLE) + target_link_libraries(easy_gui_core PUBLIC "-framework IOKit" "-framework CoreFoundation") +endif() + if(MSVC) target_compile_definitions(easy_gui_core PUBLIC _USE_MATH_DEFINES NOMINMAX) target_compile_options(easy_gui_core PRIVATE /bigobj) diff --git a/apps/easy_gui/src/batch.cpp b/apps/easy_gui/src/batch.cpp new file mode 100644 index 00000000..b4848cb1 --- /dev/null +++ b/apps/easy_gui/src/batch.cpp @@ -0,0 +1,768 @@ +#include "batch.h" + +#include "sleep_blocker.h" +#include "version.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; + +namespace easy_gui +{ + namespace + { + constexpr size_t kLogTailLines = 400; + constexpr const char* kInvalidNameCharacters = "/\\:*?\"<>|"; + + std::string timestamp() + { + const std::time_t now = std::time(nullptr); + std::tm tm{}; +#ifdef _WIN32 + localtime_s(&tm, &now); +#else + localtime_r(&now, &tm); +#endif + std::ostringstream out; + out << std::put_time(&tm, "%Y-%m-%d %H:%M:%S"); + return out.str(); + } + + std::string lowerAscii(std::string s) + { + for (char& c : s) + c = static_cast(std::tolower(static_cast(c))); + return s; + } + + bool hasRecordingFiles(const RecordingFiles& files) + { + return !files.laz_files.empty() && !files.csv_files.empty(); + } + + bool ignoredFolder(const std::string& name) + { + return name.empty() || name[0] == '.' || name[0] == '$' || name == "System Volume Information"; + } + + std::string notARecordingReason(const fs::path& folder, const RecordingFiles& files) + { + std::error_code ec; + if (fs::exists(folder / BatchRunner::kSummaryFileName, ec) || fs::exists(folder / "easy_gui.log", ec)) + return "Easy Forest HDMapping results, not a recording."; + std::string reason = files.laz_files.empty() && files.csv_files.empty() + ? "No point cloud files (lidar*.laz) and no IMU files (imu*.csv)." + : files.laz_files.empty() ? "No point cloud files (lidar*.laz)." + : "No IMU files (imu*.csv)."; + for (const auto& entry : fs::directory_iterator(folder, ec)) + if (entry.is_directory(ec) && !ignoredFolder(entry.path().filename().string())) + { + reason += " Recordings are only looked for one folder level down."; + break; + } + return reason; + } + + //! Folder name with the characters outputNameProblem() rejects replaced. + std::string sanitizedName(const std::string& name) + { + std::string out; + for (char c : name) + out += (static_cast(c) < 32 || std::strchr(kInvalidNameCharacters, c)) ? '_' : c; + while (!out.empty() && (out.back() == ' ' || out.back() == '.')) + out.pop_back(); + if (!outputNameProblem(out).empty()) + out = "recording_" + out; + return outputNameProblem(out).empty() ? out : "recording"; + } + + std::string csvField(const std::string& s) + { + if (s.find_first_of(",\"\r\n") == std::string::npos) + return s; + std::string out = "\""; + for (char c : s) + out += c == '"' ? std::string("\"\"") : std::string(1, c); + return out + "\""; + } + + std::string formatSeconds(double s) + { + std::ostringstream out; + out << std::fixed << std::setprecision(0) << s; + return out.str(); + } + + class PipelineJob : public RecordingJob + { + public: + bool start(const PipelineRequest& request, std::string& error) override + { + return pipeline_.start(request, error); + } + bool busy() const override + { + return pipeline_.busy(); + } + PipelineStatus status() const override + { + return pipeline_.status(); + } + void cancel() override + { + pipeline_.cancel(); + } + std::vector logTail() const override + { + return pipeline_.logTail(); + } + + private: + Pipeline pipeline_; + }; + } // namespace + + std::vector discoverRecordings(const fs::path& parent) + { + std::vector found; + std::error_code ec; + if (!fs::is_directory(parent, ec)) + return found; + + const RecordingFiles own = classifyRecordingFiles(parent); + if (hasRecordingFiles(own)) + { + const fs::path normal = parent.lexically_normal(); + const std::string name = normal.filename().empty() ? normal.parent_path().filename().string() : normal.filename().string(); + found.push_back({ parent, name.empty() ? std::string("recording") : name, true, {} }); + } + + std::vector subfolders; + for (const auto& entry : fs::directory_iterator(parent, ec)) + if (entry.is_directory(ec) && !ignoredFolder(entry.path().filename().string())) + subfolders.push_back(entry.path()); + std::sort(subfolders.begin(), subfolders.end()); + for (const auto& folder : subfolders) + { + const RecordingFiles files = classifyRecordingFiles(folder); + DiscoveredFolder d{ folder, folder.filename().string(), hasRecordingFiles(files), {} }; + if (!d.is_recording) + d.skip_reason = notARecordingReason(folder, files); + found.push_back(std::move(d)); + } + return found; + } + + std::string outputNameProblem(const std::string& name) + { + if (name.empty()) + return "Enter a name."; + if (name == "." || name == "..") + return "'" + name + "' cannot be used as a folder name."; + for (char c : name) + if (static_cast(c) < 32 || std::strchr(kInvalidNameCharacters, c)) + return "A folder name cannot contain / \\ : * ? \" < > |."; + if (name.back() == ' ' || name.back() == '.') + return "A folder name cannot end with a space or a dot."; + if (name.size() > 120) + return "The name is too long."; + // Device names Windows reserves, with or without an extension. + const std::string stem = lowerAscii(name.substr(0, name.find('.'))); + static const std::set kReserved = { "con", "prn", "aux", "nul", "com1", "com2", "com3", "com4", + "com5", "com6", "com7", "com8", "com9", "lpt1", "lpt2", "lpt3", + "lpt4", "lpt5", "lpt6", "lpt7", "lpt8", "lpt9" }; + if (kReserved.contains(stem)) + return "'" + name + "' is reserved by Windows."; + return {}; + } + + bool sameOutputName(const std::string& a, const std::string& b) + { + return lowerAscii(a) == lowerAscii(b); + } + + std::string recordingFingerprint(const RecordingFiles& files) + { + uint64_t bytes = 0; + std::error_code ec; + for (size_t i = 0; i < files.pairs; i++) + bytes += fs::file_size(files.laz_files[i], ec) + fs::file_size(files.csv_files[i], ec); + // The first IMU timestamp tells apart recordings of the same size (e.g. two equally long walks). + std::string first_sample; + if (!files.csv_files.empty()) + { + std::ifstream imu(files.csv_files.front()); + std::string line; + while (first_sample.empty() && std::getline(imu, line)) + if (!line.empty() && std::isdigit(static_cast(line[0]))) + first_sample = line.substr(0, line.find_first_of(" ,\t\r")); + } + return std::to_string(files.pairs) + " file pairs, " + std::to_string(bytes) + " bytes, first IMU sample " + + (first_sample.empty() ? "?" : first_sample); + } + + std::vector defaultOutputNames(const std::vector& recordings, const fs::path& output_root) + { + std::vector names; + for (const auto& r : recordings) + { + const std::string base = sanitizedName(r.name); + const std::string fingerprint = recordingFingerprint(classifyRecordingFiles(r.path)); + auto usable = [&](const std::string& candidate) + { + for (const auto& n : names) + if (sameOutputName(n, candidate)) + return false; + if (output_root.empty()) + return true; + const fs::path folder = output_root / candidate; + std::error_code ec; + if (!fs::exists(folder, ec) || fs::is_empty(folder, ec) || holdsUnfinishedOutput(folder)) + return true; + const auto marker = readDoneMarker(folder); + return marker && marker->fingerprint == fingerprint; + }; + std::string candidate = base; + for (int i = 2; !usable(candidate); i++) + candidate = base + "_" + std::to_string(i); + names.push_back(candidate); + } + return names; + } + + bool writeDoneMarker(const fs::path& output_folder, const DoneMarker& m, std::string& error) + { + const fs::path file = OutputLayout(output_folder).done_marker; + std::ofstream out(file, std::ios::binary); + out << "# Easy Forest HDMapping: this folder holds a finished result. Batch mode skips the recording when the\n" + "# same batch is started again; delete this file to process it again.\n" + << "version=" << kEasyForestVersion << "\n" + << "recording=" << m.recording << "\n" + << "fingerprint=" << m.fingerprint << "\n" + << "finished=" << m.finished << "\n" + << "duration_s=" << formatSeconds(m.duration_s) << "\n" + << "raw_points=" << m.raw_points << "\n" + << "exported_points=" << m.exported_points << "\n"; + out.close(); + if (!out) + { + error = "Could not write '" + file.string() + "'."; + return false; + } + return true; + } + + std::optional readDoneMarker(const fs::path& output_folder) + { + std::ifstream in(OutputLayout(output_folder).done_marker, std::ios::binary); + if (!in) + return std::nullopt; + DoneMarker m; + std::string line; + while (std::getline(in, line)) + { + if (!line.empty() && line.back() == '\r') + line.pop_back(); + const auto eq = line.find('='); + if (line.empty() || line[0] == '#' || eq == std::string::npos) + continue; + const std::string key = line.substr(0, eq), value = line.substr(eq + 1); + if (key == "recording") + m.recording = value; + else if (key == "fingerprint") + m.fingerprint = value; + else if (key == "finished") + m.finished = value; + else if (key == "duration_s") + m.duration_s = std::atof(value.c_str()); + else if (key == "raw_points") + m.raw_points = std::strtoull(value.c_str(), nullptr, 10); + else if (key == "exported_points") + m.exported_points = std::strtoull(value.c_str(), nullptr, 10); + } + if (m.fingerprint.empty()) + return std::nullopt; + return m; + } + + BatchDiskNeed batchDiskNeed(const std::vector& recordings) + { + BatchDiskNeed need; + for (const PreflightResult* r : recordings) + { + need.work_bytes = std::max(need.work_bytes, r->estimated_work_bytes); + need.result_bytes += r->estimated_result_bytes; + } + return need; + } + + Finding batchDiskFinding(const BatchDiskNeed& need, bool space_known, uint64_t available) + { + const std::string text = "about " + formatBytes(need.total()) + " (" + formatBytes(need.work_bytes) + + " temporary for the largest recording + " + formatBytes(need.result_bytes) + " of results for all of them)"; + if (!space_known) + return { Severity::Warning, "Could not determine the free disk space. The batch needs " + text + "." }; + if (available < need.total()) + return { Severity::Error, + "Not enough free disk space for the batch: " + text + " needed, " + formatBytes(available) + + " free. Free some space, choose an output folder on another disk, or select fewer recordings." }; + if (available < need.total() + need.total() / 2) + return { Severity::Warning, + "Free disk space is tight for the batch: " + text + " needed, " + formatBytes(available) + " free." }; + return { Severity::Ok, "Disk space for the batch: " + text + " needed, " + formatBytes(available) + " free." }; + } + + std::string batchOutputProblem(const fs::path& output_root, const std::vector& folders) + { + if (output_root.empty()) + return "Choose an output folder."; + for (const auto& f : folders) + if (f.is_recording && isSameOrInside(output_root, f.path)) + return "The output folder is inside the recording '" + f.name + + "'. Choose a folder elsewhere: Easy Forest HDMapping never writes into a recording."; + return {}; + } + + BatchCheck checkBatchRecording(const fs::path& recording, const fs::path& output_folder) + { + BatchCheck check; + check.preflight = runPreflight(recording, output_folder, true); + check.fingerprint = recordingFingerprint(check.preflight.files); + if (auto marker = readDoneMarker(output_folder); marker && marker->fingerprint == check.fingerprint) + check.done = marker; + return check; + } + + std::string firstError(const PreflightResult& result) + { + for (const auto& f : result.findings) + if (f.severity == Severity::Error) + return f.message; + return {}; + } + + const char* batchStateToken(BatchItemState state) + { + switch (state) + { + case BatchItemState::Excluded: + return "excluded"; + case BatchItemState::Pending: + return "pending"; + case BatchItemState::Running: + return "running"; + case BatchItemState::Succeeded: + return "succeeded"; + case BatchItemState::Failed: + return "failed"; + case BatchItemState::Cancelled: + return "cancelled"; + case BatchItemState::AlreadyDone: + return "already_done"; + case BatchItemState::NotRun: + return "not_run"; + } + return ""; + } + + double pipelineFraction(const PipelineStatus& s) + { + constexpr double kOdometryShare = 0.9; + if (s.state == PipelineState::Succeeded) + return 1.0; + switch (s.stage) + { + case PipelineStage::Checking: + case PipelineStage::Staging: + return 0.0; + case PipelineStage::Odometry: + return kOdometryShare * std::clamp(s.step1.overallFraction(), 0.0, 1.0); + case PipelineStage::Export: + return kOdometryShare + (1.0 - kOdometryShare) * std::clamp(s.export_fraction, 0.0, 1.0); + case PipelineStage::Cleanup: + return s.session_folder.empty() ? 0.0 : 1.0; + } + return 0.0; + } + + // ------------------------------------------------------------------------------------------------------------ + + BatchRunner::BatchRunner(RecordingJobFactory factory) + : factory_(std::move(factory)) + { + if (!factory_) + factory_ = [] + { + return std::make_unique(); + }; + } + + BatchRunner::~BatchRunner() + { + stopNow(); + wait(); + } + + bool BatchRunner::start(const BatchRequest& request, std::string& error) + { + if (busy_) + { + error = "A batch is already running."; + return false; + } + wait(); + std::error_code ec; + if (!fs::create_directories(request.output_root, ec) && ec) + { + error = "Could not create the output folder '" + request.output_root.string() + "' (" + ec.message() + ")."; + return false; + } + + request_ = request; + request_.settings.clamp(); + { + std::lock_guard lock(mutex_); + status_ = BatchStatus{}; + status_.running = true; + status_.items.resize(request_.items.size()); + std::vector known; + for (const auto& item : request_.items) + if (item.include && item.total_points > 0) + known.push_back(item.total_points); + const double average = + known.empty() ? 1.0 : static_cast(std::accumulate(known.begin(), known.end(), uint64_t{ 0 })) / known.size(); + weights_.assign(request_.items.size(), 0.0); + for (size_t i = 0; i < request_.items.size(); i++) + { + const auto& item = request_.items[i]; + status_.items[i].output_folder = request_.output_root / item.output_name; + status_.items[i].raw_points = item.total_points; + if (!item.include) + { + status_.items[i].state = BatchItemState::Excluded; + status_.items[i].error = item.exclude_reason; + } + else + weights_[i] = item.total_points > 0 ? static_cast(item.total_points) : average; + } + processing_s_ = 0.0; + processed_weight_ = 0.0; + log_tail_.clear(); + started_ = std::chrono::steady_clock::now(); + } + busy_ = true; + thread_ = std::thread(&BatchRunner::run, this); + return true; + } + + void BatchRunner::stopAfterCurrent() + { + std::lock_guard lock(mutex_); + status_.stop_after_current = true; + } + + void BatchRunner::stopNow() + { + std::lock_guard lock(mutex_); + status_.stop_after_current = true; + status_.stop_now = true; + if (job_) + job_->cancel(); + } + + void BatchRunner::wait() + { + if (thread_.joinable()) + thread_.join(); + } + + BatchStatus BatchRunner::status() const + { + std::lock_guard lock(mutex_); + BatchStatus s = status_; + if (s.running) + s.elapsed_s = std::chrono::duration(std::chrono::steady_clock::now() - started_).count(); + return s; + } + + std::vector BatchRunner::logTail() const + { + std::lock_guard lock(mutex_); + return { log_tail_.begin(), log_tail_.end() }; + } + + std::vector BatchRunner::currentLogTail() const + { + std::lock_guard lock(mutex_); + return job_ ? job_->logTail() : std::vector{}; + } + + void BatchRunner::log(const std::string& line) + { + const std::string stamped = "[" + timestamp() + "] " + line; + if (log_file_.is_open()) + log_file_ << stamped << std::endl; // flushed: the log must survive a crash or power loss mid-batch + std::lock_guard lock(mutex_); + log_tail_.push_back(stamped); + while (log_tail_.size() > kLogTailLines) + log_tail_.pop_front(); + } + + void BatchRunner::updateProgress() + { + const double total = std::accumulate(weights_.begin(), weights_.end(), 0.0); + double done = 0.0; + for (size_t i = 0; i < weights_.size(); i++) + { + const BatchItemState st = status_.items[i].state; + if (st != BatchItemState::Pending && st != BatchItemState::Running) + done += weights_[i]; + } + double current_done = 0.0, current_s = 0.0; + if (status_.current >= 0) + { + current_done = weights_[status_.current] * pipelineFraction(status_.current_status); + current_s = std::chrono::duration(std::chrono::steady_clock::now() - item_started_).count(); + } + status_.fraction = total > 0.0 ? std::clamp((done + current_done) / total, 0.0, 1.0) : 1.0; + + // ETA from the processing rate of the recordings that really ran (already-done ones take no time). + double remaining = 0.0; + for (size_t i = 0; i < weights_.size(); i++) + if (status_.items[i].state == BatchItemState::Pending) + remaining += weights_[i]; + if (status_.current >= 0) + remaining += weights_[status_.current] - current_done; + const double processed = processed_weight_ + current_done; + status_.eta_s = (total > 0.0 && processed > 0.02 * total) ? (processing_s_ + current_s) * remaining / processed : -1.0; + } + + void BatchRunner::run() + { + // Several hours of unattended processing: the computer must not go to sleep in between. + SleepBlocker no_sleep("Easy Forest HDMapping is processing recordings"); + + log_file_.open(request_.output_root / kLogFileName, std::ios::out | std::ios::app); + log("batch started: " + versionString()); + log("output folder: " + request_.output_root.string()); + log("lidar odometry program: " + request_.step1_executable.string()); + { + std::ostringstream s; + s << "settings: near_min_range_m=" << request_.settings.near_min_range_m + << " place_by_raw_time=" << request_.settings.place_by_raw_time << " keep_work_files=" << request_.settings.keep_work_files; + log(s.str()); + } + log(std::string("preventing system sleep: ") + (no_sleep.active() ? "yes" : "no (not supported or refused)")); + size_t selected = 0; + for (const auto& item : request_.items) + selected += item.include ? 1 : 0; + log(std::to_string(request_.items.size()) + " recordings found, " + std::to_string(selected) + " selected"); + for (const auto& item : request_.items) + if (!item.include) + log("excluded: " + item.name + (item.exclude_reason.empty() ? "" : " (" + item.exclude_reason + ")")); + for (const auto& folder : request_.skipped) + log("skipped: " + folder.name + " (" + folder.skip_reason + ")"); + writeSummary(); + + for (size_t i = 0; i < request_.items.size(); i++) + { + if (!request_.items[i].include) + continue; + bool stop = false; + { + std::lock_guard lock(mutex_); + stop = status_.stop_after_current; + if (stop) + status_.items[i].state = BatchItemState::NotRun; + } + if (!stop) + processItem(i); + writeSummary(); + } + + BatchStatus final_status = status(); + size_t counts[8] = {}; + for (const auto& r : final_status.items) + counts[static_cast(r.state)]++; + std::ostringstream summary; + summary << "batch " << (final_status.stop_after_current ? "stopped" : "finished") << " after " + << formatSeconds(final_status.elapsed_s) << " s: " << counts[static_cast(BatchItemState::Succeeded)] << " succeeded, " + << counts[static_cast(BatchItemState::Failed)] << " failed, " << counts[static_cast(BatchItemState::AlreadyDone)] + << " already done, " << counts[static_cast(BatchItemState::Cancelled)] << " cancelled, " + << counts[static_cast(BatchItemState::NotRun)] << " not run, " << counts[static_cast(BatchItemState::Excluded)] + << " excluded"; + log(summary.str()); + log_file_.close(); + { + std::lock_guard lock(mutex_); + status_.elapsed_s = std::chrono::duration(std::chrono::steady_clock::now() - started_).count(); + status_.running = false; + status_.current = -1; + updateProgress(); + } + busy_ = false; + } + + void BatchRunner::processItem(size_t index) + { + const BatchItem& item = request_.items[index]; + const fs::path out = request_.output_root / item.output_name; + { + std::lock_guard lock(mutex_); + status_.current = static_cast(index); + status_.current_status = PipelineStatus{}; + status_.items[index].state = BatchItemState::Running; + item_started_ = std::chrono::steady_clock::now(); + } + log("recording " + item.name + ": " + item.recording.string() + " -> " + out.string()); + + auto finishItem = [&](BatchItemState state, const std::string& error, bool ran) + { + std::lock_guard lock(mutex_); + BatchItemResult& r = status_.items[index]; + r.state = state; + r.error = error; + if (ran) + { + r.duration_s = std::chrono::duration(std::chrono::steady_clock::now() - item_started_).count(); + processing_s_ += r.duration_s; + processed_weight_ += weights_[index]; + } + status_.current = -1; + updateProgress(); + }; + + const BatchCheck check = checkBatchRecording(item.recording, out); + { + std::lock_guard lock(mutex_); + status_.items[index].raw_points = check.preflight.total_points; + } + if (check.done) + { + { + std::lock_guard lock(mutex_); + status_.items[index].duration_s = check.done->duration_s; + status_.items[index].exported_points = check.done->exported_points; + } + finishItem(BatchItemState::AlreadyDone, "", false); + log("recording " + item.name + ": already done (finished " + check.done->finished + "), skipped"); + return; + } + if (!check.preflight.ok()) + { + const std::string error = firstError(check.preflight); + finishItem(BatchItemState::Failed, error, false); + log("recording " + item.name + ": FAILED the check: " + error); + return; + } + + std::unique_ptr job = factory_(); + PipelineRequest request{ item.recording, out, request_.step1_executable, request_.settings, request_.check_band_outer_m }; + request.resume_unfinished = true; + std::string error; + if (!job->start(request, error)) + { + finishItem(BatchItemState::Failed, error, true); + log("recording " + item.name + ": FAILED to start: " + error); + return; + } + { + std::lock_guard lock(mutex_); + job_ = job.get(); + if (status_.stop_now) // stopNow() may have run before job_ was published + job_->cancel(); + } + PipelineStage last_stage = PipelineStage::Checking; + while (job->busy()) + { + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + const PipelineStatus s = job->status(); + if (s.stage != last_stage && s.state == PipelineState::Running) + { + static const char* kStageNames[] = { "checking", "preparing", "lidar odometry", "export", "cleaning up" }; + log("recording " + item.name + ": " + kStageNames[static_cast(s.stage)]); + last_stage = s.stage; + } + std::lock_guard lock(mutex_); + status_.current_status = s; + updateProgress(); + } + const PipelineStatus s = job->status(); + { + std::lock_guard lock(mutex_); + job_ = nullptr; + status_.current_status = s; + } + job.reset(); + + if (s.state == PipelineState::Succeeded) + { + const ExportStats& e = s.export_stats; + const uint64_t exported = e.odometry_points + e.near_points + e.gap_points + e.check_band_points; + DoneMarker marker{ + item.recording.string(), check.fingerprint, timestamp(), s.elapsed_s, check.preflight.total_points, exported + }; + std::string marker_error; + const bool marked = writeDoneMarker(out, marker, marker_error); + { + std::lock_guard lock(mutex_); + status_.items[index].exported_points = exported; + } + finishItem(BatchItemState::Succeeded, marked ? "" : marker_error, true); + log("recording " + item.name + ": succeeded in " + formatSeconds(s.elapsed_s) + " s, " + std::to_string(exported) + + " points exported" + (marked ? "" : "; " + marker_error)); + } + else if (s.state == PipelineState::Cancelled) + { + finishItem(BatchItemState::Cancelled, s.message, true); + log("recording " + item.name + ": cancelled"); + } + else + { + const std::string message = s.message.empty() ? "Processing failed." : s.message; + finishItem(BatchItemState::Failed, message, true); + log("recording " + item.name + ": FAILED: " + message + (s.detail.empty() ? "" : " (" + s.detail + ")")); + } + } + + void BatchRunner::writeSummary() + { + const BatchStatus s = status(); + const fs::path file = request_.output_root / kSummaryFileName; + const fs::path temporary = request_.output_root / (std::string(kSummaryFileName) + ".tmp"); + { + std::ofstream out(temporary, std::ios::binary); + out << "recording,output_name,status,duration_s,raw_points,exported_points,output_path,error\n"; + for (size_t i = 0; i < request_.items.size(); i++) + { + const BatchItem& item = request_.items[i]; + const BatchItemResult& r = s.items[i]; + std::error_code ec; + const bool has_output = r.state != BatchItemState::Excluded && fs::exists(r.output_folder, ec); + out << csvField(item.recording.string()) << ',' << csvField(item.output_name) << ',' << batchStateToken(r.state) << ',' + << (r.duration_s > 0.0 ? formatSeconds(r.duration_s) : "") << ',' << (r.raw_points ? std::to_string(r.raw_points) : "") + << ',' << (r.exported_points ? std::to_string(r.exported_points) : "") << ',' + << csvField(has_output ? r.output_folder.string() : "") << ',' << csvField(r.error) << '\n'; + } + for (const auto& folder : request_.skipped) + out << csvField(folder.path.string()) << ",,skipped,,,,," << csvField(folder.skip_reason) << '\n'; + out.close(); + if (!out) + { + log("WARNING: could not write " + temporary.string()); + return; + } + } + std::error_code ec; + fs::rename(temporary, file, ec); + if (ec) // e.g. the summary is open in a spreadsheet program on Windows + log("WARNING: could not update " + file.string() + " (" + ec.message() + ")"); + } +} // namespace easy_gui diff --git a/apps/easy_gui/src/batch.h b/apps/easy_gui/src/batch.h new file mode 100644 index 00000000..8da9dad2 --- /dev/null +++ b/apps/easy_gui/src/batch.h @@ -0,0 +1,250 @@ +#pragma once + +#include "pipeline.h" +#include "preflight.h" +#include "settings.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// Batch mode: several recordings, one after another, through the unchanged single-recording Pipeline. Each recording +// gets its own output folder // with the usual result files; the output root also holds +// batch.log and batch_summary.csv. See docs/DESIGN.md, "Batch mode". +namespace easy_gui +{ + //! A folder found by discoverRecordings(). + struct DiscoveredFolder + { + std::filesystem::path path; + std::string name; // folder name (the parent's own name when the parent itself is a recording) + bool is_recording = false; + std::string skip_reason; // plain language, when !is_recording + }; + + //! The recordings among `parent`'s direct subfolders, plus `parent` itself if it is a recording (listed first). + //! Not recursive: a recording is the folder the scanner wrote, and searching deeper would also find step 1's own + //! folders in old manual runs. A recording is a folder with at least one point cloud and one IMU file as + //! classifyRecordingFiles sees them; the pre-flight check judges the rest. Subfolders that are not recordings are + //! listed with the reason; hidden and system folders (".*", "$*", "System Volume Information") are left out. + std::vector discoverRecordings(const std::filesystem::path& parent); + + //! Why `name` cannot be used as an output folder name, or empty if it can. It must be a single folder name that + //! is valid on both Windows and macOS. + std::string outputNameProblem(const std::string& name); + + //! Output names are compared ignoring (ASCII) case: the default file systems of macOS and Windows do. + bool sameOutputName(const std::string& a, const std::string& b); + + //! Identifies a recording independently of where it is mounted (a USB stick may get another drive letter): + //! number of file pairs, the total size of the paired files and the first IMU timestamp. Reads file sizes and the + //! first lines of one IMU file only. + std::string recordingFingerprint(const RecordingFiles& files); + + //! Default output names: the folder names (characters invalid in a folder name replaced by '_'), made unique + //! ignoring case with _2, _3, ... A name is also passed over when its folder under `output_root` already holds + //! something other than this recording's finished result or an unfinished attempt. + std::vector defaultOutputNames(const std::vector& recordings, const std::filesystem::path& output_root); + + //! Contents of easy_gui_done.txt, the marker batch mode writes into a recording's output folder once its result + //! is complete. A restarted batch skips recordings whose marker matches. + struct DoneMarker + { + std::string recording; // path at the time it was processed (informative) + std::string fingerprint; // recordingFingerprint(): what identifies the recording + std::string finished; // local time + double duration_s = 0.0; + uint64_t raw_points = 0; + uint64_t exported_points = 0; + }; + + bool writeDoneMarker(const std::filesystem::path& output_folder, const DoneMarker& marker, std::string& error); + std::optional readDoneMarker(const std::filesystem::path& output_folder); + + //! Disk space a batch needs at its start: the recordings run one at a time and each frees its temporary files + //! when it ends, so only the largest temporary need counts, plus the results of all of them. + struct BatchDiskNeed + { + uint64_t work_bytes = 0; + uint64_t result_bytes = 0; + uint64_t total() const + { + return work_bytes + result_bytes; + } + }; + + BatchDiskNeed batchDiskNeed(const std::vector& recordings); + + //! The batch disk finding, with the same thresholds as the single-recording check (error below the estimate, + //! warning below 1.5x). `available` is ignored when `space_known` is false. + Finding batchDiskFinding(const BatchDiskNeed& need, bool space_known, uint64_t available); + + //! Why the batch cannot write into `output_root`, or empty: batch.log and batch_summary.csv go there, so it must + //! not be inside any recording (the per-recording check catches the recordings' own output folders). + std::string batchOutputProblem(const std::filesystem::path& output_root, const std::vector& folders); + + //! Pre-flight result of one batch recording, as shown in the table. + struct BatchCheck + { + PreflightResult preflight; // checked with resume_unfinished, against / + std::string fingerprint; + std::optional done; // set if the output folder already holds this recording's finished result + }; + + BatchCheck checkBatchRecording(const std::filesystem::path& recording, const std::filesystem::path& output_folder); + + //! First error of a pre-flight result (empty if none). + std::string firstError(const PreflightResult& result); + + // ------------------------------------------------------------------------------------------------------------ + // Running + + enum class BatchItemState + { + Excluded, // not selected, or the check failed + Pending, + Running, + Succeeded, + Failed, + Cancelled, // stopped by "stop now" while it ran + AlreadyDone, // finished in an earlier run of the same batch + NotRun, // the batch was stopped before it got there + }; + + //! Token in batch_summary.csv ("succeeded", "already_done", ...). + const char* batchStateToken(BatchItemState state); + + struct BatchItem + { + std::filesystem::path recording; + std::string name; // as discovered (shown and logged) + std::string output_name; + bool include = true; + std::string exclude_reason; // written to the summary when !include + uint64_t total_points = 0; // from the check; weights the overall progress (0 = unknown, use the average) + }; + + struct BatchRequest + { + std::filesystem::path output_root; + std::vector items; + std::vector skipped; // folders that are not recordings: only for the summary and the log + std::filesystem::path step1_executable; + Settings settings; + double check_band_outer_m = 0.0; // dev/test option, passed to every recording + }; + + struct BatchItemResult + { + BatchItemState state = BatchItemState::Pending; + double duration_s = 0.0; + uint64_t raw_points = 0; + uint64_t exported_points = 0; + std::string error; + std::filesystem::path output_folder; + }; + + struct BatchStatus + { + bool running = false; + std::vector items; // parallel to BatchRequest::items + int current = -1; // index of the item being processed + PipelineStatus current_status; // of the item being processed + double elapsed_s = 0.0; + double fraction = 0.0; // overall, 0..1, weighted by points + double eta_s = -1.0; // < 0 while unknown + bool stop_after_current = false; + bool stop_now = false; + }; + + //! Share of one recording's processing that is done, 0..1. Odometry is ~90 % of the time, the export the rest + //! (ยง9 of DESIGN.md: ~21 min odometry, ~2.5 min export). + double pipelineFraction(const PipelineStatus& status); + + //! Processes one recording. The batch runs a Pipeline; tests use a fake. + class RecordingJob + { + public: + virtual ~RecordingJob() = default; + virtual bool start(const PipelineRequest& request, std::string& error) = 0; + virtual bool busy() const = 0; + virtual PipelineStatus status() const = 0; + virtual void cancel() = 0; + virtual std::vector logTail() const + { + return {}; + } + }; + + using RecordingJobFactory = std::function()>; + + //! Runs a batch on a worker thread, one recording after another. A failed recording is marked failed and the + //! batch goes on. Each finished recording gets a done marker; recordings with a matching marker are skipped, so + //! restarting the same batch continues where it stopped. batch_summary.csv is rewritten after every recording, + //! so it is current even if the batch never ends normally. System sleep is prevented while it runs. + class BatchRunner + { + public: + //! An empty factory runs the real Pipeline. + explicit BatchRunner(RecordingJobFactory factory = {}); + ~BatchRunner(); // stops now and joins + + bool start(const BatchRequest& request, std::string& error); + //! Lets the running recording finish, then stops. + void stopAfterCurrent(); + //! Cancels the running recording (its unfinished results are removed, as in single mode) and stops. + void stopNow(); + bool busy() const + { + return busy_.load(); + } + //! Blocks until the worker thread has finished. + void wait(); + + BatchStatus status() const; + const BatchRequest& request() const + { + return request_; // only changed by start(), which refuses while busy + } + //! Last lines of batch.log, oldest first. + std::vector logTail() const; + //! Last lines of the running recording's own log. + std::vector currentLogTail() const; + + static constexpr const char* kLogFileName = "batch.log"; + static constexpr const char* kSummaryFileName = "batch_summary.csv"; + + private: + void run(); + void processItem(size_t index); + void log(const std::string& line); + void writeSummary(); + void updateProgress(); // called with mutex_ held + + RecordingJobFactory factory_; + BatchRequest request_; + std::thread thread_; + std::atomic busy_{ false }; + + mutable std::mutex mutex_; + BatchStatus status_; + std::vector weights_; // per item, for the overall fraction + double processing_s_ = 0.0; // time spent in finished recordings that really ran (for the ETA) + double processed_weight_ = 0.0; // their weight + std::chrono::steady_clock::time_point started_; + std::chrono::steady_clock::time_point item_started_; + RecordingJob* job_ = nullptr; // valid while a recording runs; guarded by mutex_ + std::deque log_tail_; + std::ofstream log_file_; // only touched by the worker thread + }; +} // namespace easy_gui diff --git a/apps/easy_gui/src/sleep_blocker.cpp b/apps/easy_gui/src/sleep_blocker.cpp new file mode 100644 index 00000000..0c2d6aed --- /dev/null +++ b/apps/easy_gui/src/sleep_blocker.cpp @@ -0,0 +1,49 @@ +#include "sleep_blocker.h" + +#if defined(__APPLE__) +#include +#include +#elif defined(_WIN32) +#include +#endif + +namespace easy_gui +{ +#if defined(__APPLE__) + SleepBlocker::SleepBlocker(const std::string& reason) + { + CFStringRef name = CFStringCreateWithCString(kCFAllocatorDefault, reason.c_str(), kCFStringEncodingUTF8); + IOPMAssertionID id = 0; + active_ = name && + IOPMAssertionCreateWithName(kIOPMAssertionTypePreventUserIdleSystemSleep, kIOPMAssertionLevelOn, name, &id) == kIOReturnSuccess; + if (name) + CFRelease(name); + assertion_id_ = id; + } + + SleepBlocker::~SleepBlocker() + { + if (active_) + IOPMAssertionRelease(assertion_id_); + } +#elif defined(_WIN32) + SleepBlocker::SleepBlocker(const std::string&) + { + active_ = SetThreadExecutionState(ES_CONTINUOUS | ES_SYSTEM_REQUIRED) != 0; + } + + SleepBlocker::~SleepBlocker() + { + if (active_) + SetThreadExecutionState(ES_CONTINUOUS); + } +#else + SleepBlocker::SleepBlocker(const std::string&) + { + } + + SleepBlocker::~SleepBlocker() + { + } +#endif +} // namespace easy_gui diff --git a/apps/easy_gui/src/sleep_blocker.h b/apps/easy_gui/src/sleep_blocker.h new file mode 100644 index 00000000..cbe49509 --- /dev/null +++ b/apps/easy_gui/src/sleep_blocker.h @@ -0,0 +1,31 @@ +#pragma once + +#include + +namespace easy_gui +{ + //! Keeps the computer from going to sleep on its own while it exists (the display may still turn off). A batch can + //! run for hours unattended, and idle sleep would suspend lidar odometry halfway. + //! - macOS: an IOKit power assertion (kIOPMAssertionTypePreventUserIdleSystemSleep), visible in `pmset -g assertions`. + //! - Windows: SetThreadExecutionState(ES_CONTINUOUS | ES_SYSTEM_REQUIRED). It applies to the calling thread, so a + //! SleepBlocker must be destroyed on the thread that created it. + //! - elsewhere: does nothing. + class SleepBlocker + { + public: + explicit SleepBlocker(const std::string& reason); + ~SleepBlocker(); + SleepBlocker(const SleepBlocker&) = delete; + SleepBlocker& operator=(const SleepBlocker&) = delete; + + //! Whether the system accepted the request. + bool active() const + { + return active_; + } + + private: + bool active_ = false; + unsigned int assertion_id_ = 0; // macOS IOPMAssertionID + }; +} // namespace easy_gui diff --git a/apps/easy_gui/tests/CMakeLists.txt b/apps/easy_gui/tests/CMakeLists.txt index ed521715..82eb523e 100644 --- a/apps/easy_gui/tests/CMakeLists.txt +++ b/apps/easy_gui/tests/CMakeLists.txt @@ -9,6 +9,7 @@ project(easy_gui_tests) # replays an excerpt of a real lidar_odometry_step_1 log. add_executable(easy_gui_tests test_main.cpp + test_batch.cpp test_child_process.cpp test_export.cpp test_pipeline_integration.cpp diff --git a/apps/easy_gui/tests/test_batch.cpp b/apps/easy_gui/tests/test_batch.cpp new file mode 100644 index 00000000..8cd06168 --- /dev/null +++ b/apps/easy_gui/tests/test_batch.cpp @@ -0,0 +1,574 @@ +#include + +#include "batch.h" +#include "test_helpers.h" + +#include +#include +#include + +using namespace easy_gui; +using easy_gui_test::TempDir; +namespace fs = std::filesystem; + +namespace +{ + std::vector readLines(const fs::path& file) + { + std::ifstream in(file); + std::vector lines; + std::string line; + while (std::getline(in, line)) + lines.push_back(line); + return lines; + } + + const DiscoveredFolder* findFolder(const std::vector& folders, const std::string& name) + { + for (const auto& f : folders) + if (f.name == name) + return &f; + return nullptr; + } + + //! Stands in for the Pipeline: recordings whose folder name contains "fail" fail, the others succeed and leave + //! a result file. With `block` set, a job stays busy until release() or cancel(). + struct FakeJobs + { + std::mutex mutex; + std::condition_variable cv; + std::vector started; + bool block = false; + bool released = false; + + void release() + { + std::lock_guard lock(mutex); + released = true; + cv.notify_all(); + } + + size_t startedCount() + { + std::lock_guard lock(mutex); + return started.size(); + } + + RecordingJobFactory factory() + { + return [this] + { + return std::make_unique(*this); + }; + } + + class Job : public RecordingJob + { + public: + explicit Job(FakeJobs& owner) + : owner_(owner) + { + } + ~Job() override + { + if (thread_.joinable()) + thread_.join(); + } + bool start(const PipelineRequest& request, std::string&) override + { + { + std::lock_guard lock(owner_.mutex); + owner_.started.push_back(request); + } + status_.state = PipelineState::Running; + status_.stage = PipelineStage::Odometry; + busy_ = true; + thread_ = std::thread( + [this, request] + { + bool cancelled = false; + { + std::unique_lock lock(owner_.mutex); + owner_.cv.wait( + lock, + [&] + { + return !owner_.block || owner_.released || cancel_; + }); + cancelled = cancel_; + } + std::lock_guard lock(status_mutex_); + if (cancelled) + { + status_.state = PipelineState::Cancelled; + status_.message = "Processing was cancelled."; + } + else if (request.recording.filename().string().find("fail") != std::string::npos) + { + status_.state = PipelineState::Failed; + status_.message = "Lidar odometry stopped unexpectedly (exit code 3)."; + } + else + { + fs::create_directories(request.output_folder); + easy_gui_test::writeText(request.output_folder / "point_cloud.laz", "fake"); + status_.state = PipelineState::Succeeded; + status_.export_stats.odometry_points = 1000; + status_.export_stats.near_points = 20; + status_.export_stats.gap_points = 3; + status_.elapsed_s = 12.0; + } + busy_ = false; + }); + return true; + } + bool busy() const override + { + return busy_; + } + PipelineStatus status() const override + { + std::lock_guard lock(status_mutex_); + return status_; + } + void cancel() override + { + std::lock_guard lock(owner_.mutex); + cancel_ = true; + owner_.cv.notify_all(); + } + + private: + FakeJobs& owner_; + std::thread thread_; + std::atomic busy_{ false }; + bool cancel_ = false; // guarded by owner_.mutex + mutable std::mutex status_mutex_; + PipelineStatus status_; + }; + }; + + //! What the GUI and --batch do before starting: discover, name, check, select the recordings that passed. + BatchRequest prepareBatch(const fs::path& parent, const fs::path& output_root) + { + BatchRequest request; + request.output_root = output_root; + std::vector recordings; + for (const auto& f : discoverRecordings(parent)) + (f.is_recording ? recordings : request.skipped).push_back(f); + const auto names = defaultOutputNames(recordings, output_root); + for (size_t i = 0; i < recordings.size(); i++) + { + const BatchCheck check = checkBatchRecording(recordings[i].path, output_root / names[i]); + BatchItem item{ recordings[i].path, recordings[i].name, names[i] }; + item.total_points = check.preflight.total_points; + if (!check.done && !check.preflight.ok()) + { + item.include = false; + item.exclude_reason = firstError(check.preflight); + } + request.items.push_back(item); + } + return request; + } + + BatchStatus runToEnd(BatchRunner& runner, const BatchRequest& request) + { + std::string error; + REQUIRE(runner.start(request, error)); + runner.wait(); + return runner.status(); + } + + BatchStatus waitFor(const BatchRunner& runner, bool (*done)(const BatchStatus&)) + { + for (int i = 0; i < 200; i++) + { + const BatchStatus s = runner.status(); + if (done(s)) + return s; + std::this_thread::sleep_for(std::chrono::milliseconds(25)); + } + return runner.status(); + } +} // namespace + +TEST_CASE("discoverRecordings finds recordings one level down and explains the other folders") +{ + TempDir tmp; + const fs::path parent = tmp.path / "field_day"; + easy_gui_test::writeRecording(parent / "plot_b"); + easy_gui_test::writeRecording(parent / "plot_a"); + fs::create_directories(parent / "notes"); + easy_gui_test::writeText(parent / "notes" / "todo.txt", "x"); + easy_gui_test::writeRecording(parent / "day2" / "plot_c"); // nested two levels: not searched + fs::create_directories(parent / "only_imu"); + easy_gui_test::writeImuCsv(parent / "only_imu" / "imu0000.csv", 10, 1783328629.0); + easy_gui_test::writeRecording(parent / ".Trashes"); // hidden: ignored even though it looks like a recording + easy_gui_test::writeText(parent / "readme.txt", "a file, not a folder"); + + const auto found = discoverRecordings(parent); + REQUIRE(found.size() == 5); + CHECK(found[0].name == "day2"); // sorted by name + CHECK_FALSE(found[0].is_recording); + CHECK(found[0].skip_reason.find("one folder level down") != std::string::npos); + CHECK(findFolder(found, "notes")->skip_reason.find("No point cloud files") != std::string::npos); + CHECK(findFolder(found, "only_imu")->skip_reason == "No point cloud files (lidar*.laz)."); + CHECK(findFolder(found, "plot_a")->is_recording); + CHECK(findFolder(found, "plot_b")->is_recording); + CHECK(findFolder(found, ".Trashes") == nullptr); + CHECK(findFolder(found, "plot_c") == nullptr); + + CHECK(discoverRecordings(tmp.path / "missing").empty()); +} + +TEST_CASE("discoverRecordings accepts the parent itself as a recording") +{ + TempDir tmp; + const fs::path parent = tmp.path / "continousScanning_0012"; + easy_gui_test::writeRecording(parent); + SUBCASE("alone") + { + const auto found = discoverRecordings(parent); + REQUIRE(found.size() == 1); + CHECK(found[0].is_recording); + CHECK(found[0].name == "continousScanning_0012"); + CHECK(found[0].path == parent); + } + SUBCASE("with a trailing separator and a recording inside it, listed first") + { + easy_gui_test::writeRecording(parent / "continousScanning_0012"); + const auto found = discoverRecordings(parent.string() + "/"); + REQUIRE(found.size() == 2); + CHECK(found[0].name == "continousScanning_0012"); + CHECK(found[1].name == "continousScanning_0012"); + CHECK(found[1].path.filename() == "continousScanning_0012"); + + // The same name twice: the default output names differ. + const auto names = defaultOutputNames(found, tmp.path / "out"); + CHECK(names == std::vector{ "continousScanning_0012", "continousScanning_0012_2" }); + } +} + +TEST_CASE("discoverRecordings recognises Easy Forest HDMapping results inside the parent") +{ + TempDir tmp; + easy_gui_test::writeRecording(tmp.path / "rec"); + fs::create_directories(tmp.path / "results"); + easy_gui_test::writeText(tmp.path / "results" / "batch_summary.csv", "x"); + const auto found = discoverRecordings(tmp.path); + CHECK(findFolder(found, "results")->skip_reason == "Easy Forest HDMapping results, not a recording."); +} + +TEST_CASE("output names: validation and collisions") +{ + CHECK(outputNameProblem("plot_12").empty()); + CHECK(outputNameProblem("Plot 12 (north)").empty()); + CHECK_FALSE(outputNameProblem("").empty()); + CHECK_FALSE(outputNameProblem("a/b").empty()); + CHECK_FALSE(outputNameProblem("a\\b").empty()); + CHECK_FALSE(outputNameProblem("a:b").empty()); + CHECK_FALSE(outputNameProblem("..").empty()); + CHECK_FALSE(outputNameProblem("plot.").empty()); + CHECK_FALSE(outputNameProblem("plot ").empty()); + CHECK_FALSE(outputNameProblem("CON").empty()); + CHECK_FALSE(outputNameProblem("com1.txt").empty()); + CHECK(outputNameProblem("console").empty()); + CHECK(sameOutputName("Plot_A", "plot_a")); + CHECK_FALSE(sameOutputName("plot_a", "plot_b")); + + TempDir tmp; + easy_gui_test::writeRecording(tmp.path / "x1"); + easy_gui_test::writeRecording(tmp.path / "x2", 1783329629.0); + easy_gui_test::writeRecording(tmp.path / "x3", 1783330629.0); + const fs::path out = tmp.path / "out"; + + SUBCASE("names that differ only in case get a suffix; invalid characters are replaced") + { + const std::vector recordings = { { tmp.path / "x1", "Plot", true, {} }, + { tmp.path / "x2", "plot", true, {} }, + { tmp.path / "x3", "a:b.", true, {} } }; + CHECK(defaultOutputNames(recordings, out) == std::vector{ "Plot", "plot_2", "a_b" }); + } + SUBCASE("a folder holding something else is passed over; this recording's result and unfinished attempts are reused") + { + fs::create_directories(out / "x1"); + easy_gui_test::writeText(out / "x1" / "my_notes.txt", "not ours"); + fs::create_directories(out / "x2"); + easy_gui_test::writeText(out / "x2" / "easy_gui.log", "unfinished attempt"); + fs::create_directories(out / "x3"); + std::string error; + DoneMarker marker; + marker.fingerprint = recordingFingerprint(classifyRecordingFiles(tmp.path / "x3")); + REQUIRE(writeDoneMarker(out / "x3", marker, error)); + const std::vector recordings = { { tmp.path / "x1", "x1", true, {} }, + { tmp.path / "x2", "x2", true, {} }, + { tmp.path / "x3", "x3", true, {} } }; + CHECK(defaultOutputNames(recordings, out) == std::vector{ "x1_2", "x2", "x3" }); + + // ... but another recording's finished result is not reused. + const std::vector other = { { tmp.path / "x1", "x3", true, {} } }; + CHECK(defaultOutputNames(other, out) == std::vector{ "x3_2" }); + } +} + +TEST_CASE("done markers round-trip and identify the recording by its files, not its path") +{ + TempDir tmp; + easy_gui_test::writeRecording(tmp.path / "rec"); + const auto files = classifyRecordingFiles(tmp.path / "rec"); + const std::string fingerprint = recordingFingerprint(files); + CHECK(fingerprint.find("2 file pairs") == 0); + + fs::create_directories(tmp.path / "out"); + DoneMarker m{ "/Volumes/USB/rec", fingerprint, "2026-10-18 10:00:00", 1234.0, 4000, 3900 }; + std::string error; + REQUIRE(writeDoneMarker(tmp.path / "out", m, error)); + const auto r = readDoneMarker(tmp.path / "out"); + REQUIRE(r); + CHECK(r->recording == m.recording); + CHECK(r->fingerprint == fingerprint); + CHECK(r->finished == m.finished); + CHECK(r->duration_s == 1234.0); + CHECK(r->raw_points == 4000); + CHECK(r->exported_points == 3900); + CHECK_FALSE(readDoneMarker(tmp.path / "nowhere")); + + // The same files at another path (copied off the USB stick) are the same recording. + fs::copy(tmp.path / "rec", tmp.path / "copy"); + CHECK(checkBatchRecording(tmp.path / "copy", tmp.path / "out").done.has_value()); + // A different recording is not. + easy_gui_test::writeRecording(tmp.path / "other"); + easy_gui_test::writeRawLaz(tmp.path / "other" / "lidar0001.laz", 2100, 1783328629.0 + 30.0); + CHECK_FALSE(checkBatchRecording(tmp.path / "other", tmp.path / "out").done.has_value()); +} + +TEST_CASE("the batch disk check counts the largest temporary need once and every result") +{ + PreflightResult a, b, c; + a.estimated_work_bytes = 10'000'000'000; + a.estimated_result_bytes = 2'000'000'000; + b.estimated_work_bytes = 14'000'000'000; + b.estimated_result_bytes = 3'000'000'000; + c.estimated_work_bytes = 6'000'000'000; + c.estimated_result_bytes = 1'000'000'000; + const BatchDiskNeed need = batchDiskNeed({ &a, &b, &c }); + CHECK(need.work_bytes == 14'000'000'000); + CHECK(need.result_bytes == 6'000'000'000); + CHECK(need.total() == 20'000'000'000); + CHECK(batchDiskNeed({}).total() == 0); + + CHECK(batchDiskFinding(need, true, 19'000'000'000).severity == Severity::Error); + CHECK(batchDiskFinding(need, true, 25'000'000'000).severity == Severity::Warning); + const Finding ok = batchDiskFinding(need, true, 40'000'000'000); + CHECK(ok.severity == Severity::Ok); + CHECK(ok.message.find("14.0 GB temporary for the largest recording + 6.0 GB of results") != std::string::npos); + CHECK(batchDiskFinding(need, false, 0).severity == Severity::Warning); +} + +TEST_CASE("the batch output folder must not be inside a recording") +{ + TempDir tmp; + easy_gui_test::writeRecording(tmp.path / "parent"); + easy_gui_test::writeRecording(tmp.path / "parent" / "sub"); + const auto found = discoverRecordings(tmp.path / "parent"); + CHECK_FALSE(batchOutputProblem(tmp.path / "parent" / "results", found).empty()); // the parent is a recording + CHECK_FALSE(batchOutputProblem(tmp.path / "parent" / "sub" / "out", found).empty()); + CHECK(batchOutputProblem(tmp.path / "results", found).empty()); + CHECK_FALSE(batchOutputProblem({}, found).empty()); + + // A plain container parent may hold the output folder. + easy_gui_test::writeRecording(tmp.path / "container" / "rec"); + CHECK(batchOutputProblem(tmp.path / "container" / "results", discoverRecordings(tmp.path / "container")).empty()); +} + +TEST_CASE("a failed recording does not stop the batch; the summary and the log record every folder") +{ + TempDir tmp; + const fs::path parent = tmp.path / "parent"; + easy_gui_test::writeRecording(parent / "a_plot"); + easy_gui_test::writeRecording(parent / "b_fail", 1783329629.0); + easy_gui_test::writeRecording(parent / "c_plot, north", 1783330629.0); // comma: quoted in the CSV + fs::create_directories(parent / "d_damaged"); + easy_gui_test::writeText(parent / "d_damaged" / "lidar0000.laz", "not a LAZ file"); + easy_gui_test::writeImuCsv(parent / "d_damaged" / "imu0000.csv", 6000, 1783328629.0); + fs::create_directories(parent / "e_notes"); + const fs::path out = tmp.path / "out"; + + BatchRequest request = prepareBatch(parent, out); + REQUIRE(request.items.size() == 4); + CHECK_FALSE(request.items[3].include); // d_damaged fails the check and is not selected + CHECK(request.items[3].exclude_reason.find("cannot be read") != std::string::npos); + REQUIRE(request.skipped.size() == 1); + + FakeJobs jobs; + BatchRunner runner(jobs.factory()); + const BatchStatus s = runToEnd(runner, request); + CHECK_FALSE(s.running); + CHECK(s.items[0].state == BatchItemState::Succeeded); + CHECK(s.items[1].state == BatchItemState::Failed); + CHECK(s.items[1].error == "Lidar odometry stopped unexpectedly (exit code 3)."); + CHECK(s.items[2].state == BatchItemState::Succeeded); + CHECK(s.items[3].state == BatchItemState::Excluded); + CHECK(s.items[0].exported_points == 1023); + CHECK(s.items[0].raw_points == 4000); + CHECK(s.fraction == doctest::Approx(1.0)); + REQUIRE(jobs.started.size() == 3); + CHECK(jobs.started[0].output_folder == out / "a_plot"); + CHECK(jobs.started[0].resume_unfinished); + + CHECK(readDoneMarker(out / "a_plot")); + CHECK_FALSE(readDoneMarker(out / "b_fail")); + CHECK(readDoneMarker(out / "c_plot, north")); + + const auto csv = readLines(out / BatchRunner::kSummaryFileName); + REQUIRE(csv.size() == 6); + CHECK(csv[0] == "recording,output_name,status,duration_s,raw_points,exported_points,output_path,error"); + CHECK(csv[1].find(",a_plot,succeeded,") != std::string::npos); + CHECK(csv[1].find(",4000,1023,") != std::string::npos); + CHECK(csv[2].find(",b_fail,failed,") != std::string::npos); + CHECK(csv[2].find("exit code 3") != std::string::npos); + CHECK(csv[3].find(",\"c_plot, north\",succeeded,") != std::string::npos); + CHECK(csv[4].find(",d_damaged,excluded,") != std::string::npos); + CHECK(csv[5].find(",,skipped,,,,,") != std::string::npos); + CHECK_FALSE(fs::exists(out / "batch_summary.csv.tmp")); + + const auto log = readLines(out / BatchRunner::kLogFileName); + auto logHas = [&](const std::string& text) + { + return std::any_of( + log.begin(), + log.end(), + [&](const std::string& l) + { + return l.find(text) != std::string::npos; + }); + }; + CHECK(logHas("batch started")); + CHECK(logHas("recording b_fail: FAILED")); + CHECK(logHas("skipped: e_notes")); + CHECK(logHas("excluded: d_damaged")); + CHECK(logHas("1 failed")); +} + +TEST_CASE("restarting the same batch skips finished recordings and retries the others") +{ + TempDir tmp; + const fs::path parent = tmp.path / "parent"; + easy_gui_test::writeRecording(parent / "a_plot"); + easy_gui_test::writeRecording(parent / "b_fail", 1783329629.0); + easy_gui_test::writeRecording(parent / "c_plot", 1783330629.0); + const fs::path out = tmp.path / "out"; + { + FakeJobs jobs; + BatchRunner runner(jobs.factory()); + runToEnd(runner, prepareBatch(parent, out)); + REQUIRE(jobs.started.size() == 3); + } + // b_fail left an unfinished attempt (a log and a half-written point cloud); it is retried, not refused. + fs::create_directories(out / "b_fail"); + easy_gui_test::writeText(out / "b_fail" / "easy_gui.log", "first attempt\n"); + easy_gui_test::writeText(out / "b_fail" / "point_cloud.laz", "partial"); + + const BatchRequest again = prepareBatch(parent, out); + REQUIRE(again.items.size() == 3); + CHECK(again.items[0].output_name == "a_plot"); // same names as before, so the markers are found + CHECK(again.items[1].include); + + FakeJobs jobs; + BatchRunner runner(jobs.factory()); + const BatchStatus s = runToEnd(runner, again); + CHECK(s.items[0].state == BatchItemState::AlreadyDone); + CHECK(s.items[0].exported_points == 1023); // from the marker + CHECK(s.items[1].state == BatchItemState::Failed); + CHECK(s.items[2].state == BatchItemState::AlreadyDone); + REQUIRE(jobs.started.size() == 1); + CHECK(jobs.started[0].recording == parent / "b_fail"); + const auto csv = readLines(out / BatchRunner::kSummaryFileName); + CHECK(csv[1].find(",a_plot,already_done,") != std::string::npos); +} + +TEST_CASE("a recording whose output folder holds another recording's result fails and the batch continues") +{ + TempDir tmp; + const fs::path parent = tmp.path / "parent"; + easy_gui_test::writeRecording(parent / "a_plot"); + easy_gui_test::writeRecording(parent / "b_plot", 1783329629.0); + const fs::path out = tmp.path / "out"; + BatchRequest request = prepareBatch(parent, out); + // Simulates a stale table: the folder was filled after the check. + fs::create_directories(out / "a_plot"); + easy_gui_test::writeText(out / "a_plot" / "my_notes.txt", "not ours"); + + FakeJobs jobs; + BatchRunner runner(jobs.factory()); + const BatchStatus s = runToEnd(runner, request); + CHECK(s.items[0].state == BatchItemState::Failed); + CHECK(s.items[0].error.find("already contains files") != std::string::npos); + CHECK(s.items[1].state == BatchItemState::Succeeded); + CHECK(jobs.startedCount() == 1); +} + +TEST_CASE("stop after current lets the running recording finish; stop now cancels it") +{ + TempDir tmp; + const fs::path parent = tmp.path / "parent"; + easy_gui_test::writeRecording(parent / "a_plot"); + easy_gui_test::writeRecording(parent / "b_plot", 1783329629.0); + easy_gui_test::writeRecording(parent / "c_plot", 1783330629.0); + const fs::path out = tmp.path / "out"; + + FakeJobs jobs; + jobs.block = true; + BatchRunner runner(jobs.factory()); + std::string error; + REQUIRE(runner.start(prepareBatch(parent, out), error)); + const BatchStatus running = waitFor( + runner, + [](const BatchStatus& s) + { + return s.current == 0 && s.current_status.state == PipelineState::Running; + }); + REQUIRE(running.current == 0); + CHECK(runner.busy()); + + SUBCASE("stop after current") + { + runner.stopAfterCurrent(); + CHECK(runner.status().stop_after_current); + jobs.release(); + runner.wait(); + const BatchStatus s = runner.status(); + CHECK(s.items[0].state == BatchItemState::Succeeded); + CHECK(s.items[1].state == BatchItemState::NotRun); + CHECK(s.items[2].state == BatchItemState::NotRun); + CHECK(readDoneMarker(out / "a_plot")); + } + SUBCASE("stop now") + { + runner.stopNow(); + runner.wait(); + const BatchStatus s = runner.status(); + CHECK(s.items[0].state == BatchItemState::Cancelled); + CHECK(s.items[1].state == BatchItemState::NotRun); + CHECK(s.items[2].state == BatchItemState::NotRun); + CHECK_FALSE(readDoneMarker(out / "a_plot")); + const auto csv = readLines(out / BatchRunner::kSummaryFileName); + CHECK(csv[1].find(",a_plot,cancelled,") != std::string::npos); + CHECK(csv[2].find(",b_plot,not_run,") != std::string::npos); + } + CHECK(jobs.startedCount() == 1); +} + +TEST_CASE("pipelineFraction weights odometry and export") +{ + PipelineStatus s; + s.state = PipelineState::Running; + s.stage = PipelineStage::Checking; + CHECK(pipelineFraction(s) == 0.0); + s.stage = PipelineStage::Export; + s.export_fraction = 0.5; + CHECK(pipelineFraction(s) == doctest::Approx(0.95)); + s.state = PipelineState::Succeeded; + CHECK(pipelineFraction(s) == 1.0); +} From 45792e90f2ad7e31509c3809cce001a6c21c29b9 Mon Sep 17 00:00:00 2001 From: Martin Mokros Date: Sun, 18 Oct 2026 15:06:42 +0100 Subject: [PATCH 20/23] easy_gui: --batch Headless batch mode with the GUI's rules: every recording that passes the check is processed, finished ones are skipped, the batch disk check refuses to start when space is short. Ctrl+C stops after the current recording, a second Ctrl+C stops now. Exit code 0 when every selected recording succeeded or was already done. planBatch() (discover, name, check) is shared by --batch and the tests. The integration test runs two copies of the test recording plus a recording that passes the check but fails in lidar odometry, then restarts the batch. Also moves the stage progress bar and the advanced settings of the window into their own functions, for the batch view. Co-Authored-By: Claude Opus 5.5 --- apps/easy_gui/src/batch.cpp | 34 ++ apps/easy_gui/src/batch.h | 15 + apps/easy_gui/src/easy_gui_main.cpp | 299 ++++++++++++++---- apps/easy_gui/tests/test_batch.cpp | 21 +- .../tests/test_pipeline_integration.cpp | 113 ++++++- 5 files changed, 396 insertions(+), 86 deletions(-) diff --git a/apps/easy_gui/src/batch.cpp b/apps/easy_gui/src/batch.cpp index b4848cb1..99849eb3 100644 --- a/apps/easy_gui/src/batch.cpp +++ b/apps/easy_gui/src/batch.cpp @@ -348,6 +348,40 @@ namespace easy_gui return {}; } + std::vector BatchPlan::toProcess() const + { + std::vector out; + for (size_t i = 0; i < checks.size(); i++) + if (request.items[i].include && !checks[i].done) + out.push_back(&checks[i].preflight); + return out; + } + + BatchPlan planBatch(const fs::path& parent, const fs::path& output_root) + { + BatchPlan plan; + plan.found = discoverRecordings(parent); + plan.request.output_root = output_root; + std::vector recordings; + for (const auto& f : plan.found) + (f.is_recording ? recordings : plan.request.skipped).push_back(f); + const auto names = defaultOutputNames(recordings, output_root); + for (size_t i = 0; i < recordings.size(); i++) + { + plan.checks.push_back(checkBatchRecording(recordings[i].path, output_root / names[i])); + const BatchCheck& check = plan.checks.back(); + BatchItem item{ recordings[i].path, recordings[i].name, names[i] }; + item.total_points = check.preflight.total_points; + if (!check.done && !check.preflight.ok()) + { + item.include = false; + item.exclude_reason = "Check failed: " + firstError(check.preflight); + } + plan.request.items.push_back(std::move(item)); + } + return plan; + } + const char* batchStateToken(BatchItemState state) { switch (state) diff --git a/apps/easy_gui/src/batch.h b/apps/easy_gui/src/batch.h index 8da9dad2..bf6b2109 100644 --- a/apps/easy_gui/src/batch.h +++ b/apps/easy_gui/src/batch.h @@ -192,6 +192,21 @@ namespace easy_gui //! batch goes on. Each finished recording gets a done marker; recordings with a matching marker are skipped, so //! restarting the same batch continues where it stopped. batch_summary.csv is rewritten after every recording, //! so it is current even if the batch never ends normally. System sleep is prevented while it runs. + //! What happens before "Start", done synchronously (for --batch and tests; the GUI checks row by row in the + //! background): discover, name, check. Recordings that fail the check are not selected (exclude_reason says + //! why); finished ones stay selected, so the summary lists them as already done. + struct BatchPlan + { + std::vector found; // everything discoverRecordings() returned + BatchRequest request; // items + skipped filled; the caller sets step1_executable and the settings + std::vector checks; // parallel to request.items + + //! Pre-flight results of the recordings that will really be processed (selected, not yet done). + std::vector toProcess() const; + }; + + BatchPlan planBatch(const std::filesystem::path& parent, const std::filesystem::path& output_root); + class BatchRunner { public: diff --git a/apps/easy_gui/src/easy_gui_main.cpp b/apps/easy_gui/src/easy_gui_main.cpp index df701327..31afff27 100644 --- a/apps/easy_gui/src/easy_gui_main.cpp +++ b/apps/easy_gui/src/easy_gui_main.cpp @@ -1,6 +1,7 @@ // HDMapping Easy GUI: raw Mandeye recording -> lidar odometry (precision forestry preset) -> point cloud + trajectory. // See apps/easy_gui/docs/DESIGN.md for the decisions behind the pipeline. +#include "batch.h" #include "child_process.h" #include "export.h" #include "pipeline.h" @@ -17,6 +18,7 @@ #include #include +#include #include #include #include @@ -107,7 +109,7 @@ namespace preflight.reset(); preflight_for_recording = recording; preflight_for_output = output_folder; - preflight_future = std::async(std::launch::async, runPreflight, recording, output_folder); + preflight_future = std::async(std::launch::async, runPreflight, recording, output_folder, false); } bool checking() const @@ -128,6 +130,42 @@ namespace } }; + void drawAdvancedSettings(Settings& settings) + { + if (ImGui::CollapsingHeader("Advanced settings")) + { + ImGui::Indent(); + float near = static_cast(settings.near_min_range_m); + ImGui::SetNextItemWidth(220); + if (ImGui::SliderFloat( + "Closest points kept [m]", + &near, + static_cast(Settings::kNearMinRangeLowest), + static_cast(Settings::kNearMinRangeHighest), + "%.2f m")) + { + settings.near_min_range_m = near; + settings.clamp(); + } + ImGui::SameLine(); + ImGui::TextDisabled("(?)"); + if (ImGui::IsItemHovered()) + ImGui::SetTooltip( + "Points closer to the scanner than this are left out of the point cloud.\n" + "0.4 m (default) removes the operator's body and the handle.\n" + "Lower values keep more of the close surroundings but may include the operator."); + ImGui::Checkbox("Place all points by their measurement time (experimental)", &settings.place_by_raw_time); + ImGui::SameLine(); + ImGui::TextDisabled("(?)"); + if (ImGui::IsItemHovered()) + ImGui::SetTooltip( + "Off (default): the point cloud uses lidar odometry's own points.\n" + "On: every point is taken from the raw recording and placed at its exact measurement time."); + ImGui::Checkbox("Keep intermediate files (for troubleshooting)", &settings.keep_work_files); + ImGui::Unindent(); + } + } + void drawInputs(App& app, bool locked) { ImGui::BeginDisabled(locked); @@ -160,38 +198,7 @@ namespace ImGui::TextUnformatted( app.output_folder.empty() ? "(set automatically once a recording is chosen)" : app.output_folder.string().c_str()); - if (ImGui::CollapsingHeader("Advanced settings")) - { - ImGui::Indent(); - float near = static_cast(app.settings.near_min_range_m); - ImGui::SetNextItemWidth(220); - if (ImGui::SliderFloat( - "Closest points kept [m]", - &near, - static_cast(Settings::kNearMinRangeLowest), - static_cast(Settings::kNearMinRangeHighest), - "%.2f m")) - { - app.settings.near_min_range_m = near; - app.settings.clamp(); - } - ImGui::SameLine(); - ImGui::TextDisabled("(?)"); - if (ImGui::IsItemHovered()) - ImGui::SetTooltip( - "Points closer to the scanner than this are left out of the point cloud.\n" - "0.4 m (default) removes the operator's body and the handle.\n" - "Lower values keep more of the close surroundings but may include the operator."); - ImGui::Checkbox("Place all points by their measurement time (experimental)", &app.settings.place_by_raw_time); - ImGui::SameLine(); - ImGui::TextDisabled("(?)"); - if (ImGui::IsItemHovered()) - ImGui::SetTooltip( - "Off (default): the point cloud uses lidar odometry's own points.\n" - "On: every point is taken from the raw recording and placed at its exact measurement time."); - ImGui::Checkbox("Keep intermediate files (for troubleshooting)", &app.settings.keep_work_files); - ImGui::Unindent(); - } + drawAdvancedSettings(app.settings); ImGui::EndDisabled(); } @@ -245,6 +252,33 @@ namespace ImGui::TextUnformatted(label); } + //! What the running pipeline is doing right now: a label and a progress bar. + void drawStageProgress(const PipelineStatus& s) + { + if (s.stage == PipelineStage::Odometry) + { + const double phase = s.step1.phaseFraction(); + std::string label = phaseLabel(s.step1.phase); + if (s.step1.phase == Step1Phase::Odometry && s.step1.chunks_total > 0) + label += " (" + std::to_string(s.step1.chunks_done) + " / " + std::to_string(s.step1.chunks_total) + ")"; + ImGui::TextUnformatted(label.c_str()); + if (phase >= 0.0) + ImGui::ProgressBar(static_cast(phase), ImVec2(-1, 0)); + else + ImGui::ProgressBar(-1.0f * static_cast(ImGui::GetTime()), ImVec2(-1, 0), "working..."); + } + else if (s.stage == PipelineStage::Export) + { + ImGui::TextUnformatted(s.export_step.empty() ? "Exporting..." : s.export_step.c_str()); + ImGui::ProgressBar(static_cast(s.export_fraction), ImVec2(-1, 0)); + } + else + { + ImGui::TextUnformatted(s.stage == PipelineStage::Cleanup ? "Cleaning up..." : "Preparing..."); + ImGui::ProgressBar(-1.0f * static_cast(ImGui::GetTime()), ImVec2(-1, 0), "working..."); + } + } + void drawRun(App& app) { const PipelineStatus s = app.pipeline.status(); @@ -281,18 +315,9 @@ namespace if (s.state == PipelineState::Running) { ImGui::Spacing(); + drawStageProgress(s); if (s.stage == PipelineStage::Odometry) { - const double phase = s.step1.phaseFraction(); - std::string label = phaseLabel(s.step1.phase); - if (s.step1.phase == Step1Phase::Odometry && s.step1.chunks_total > 0) - label += " (" + std::to_string(s.step1.chunks_done) + " / " + std::to_string(s.step1.chunks_total) + ")"; - ImGui::TextUnformatted(label.c_str()); - if (phase >= 0.0) - ImGui::ProgressBar(static_cast(phase), ImVec2(-1, 0)); - else - ImGui::ProgressBar(-1.0f * static_cast(ImGui::GetTime()), ImVec2(-1, 0), "working..."); - const double overall = s.step1.overallFraction(); std::string eta = "Elapsed " + formatDuration(s.elapsed_s); if (overall > 0.05) @@ -301,16 +326,7 @@ namespace ImGui::ProgressBar(static_cast(overall), ImVec2(-1, 0), eta.c_str()); } else if (s.stage == PipelineStage::Export) - { - ImGui::TextUnformatted(s.export_step.empty() ? "Exporting..." : s.export_step.c_str()); - ImGui::ProgressBar(static_cast(s.export_fraction), ImVec2(-1, 0)); ImGui::TextUnformatted(("Elapsed " + formatDuration(s.elapsed_s)).c_str()); - } - else - { - ImGui::TextUnformatted(s.stage == PipelineStage::Cleanup ? "Cleaning up..." : "Preparing..."); - ImGui::ProgressBar(-1.0f * static_cast(ImGui::GetTime()), ImVec2(-1, 0), "working..."); - } ImGui::Spacing(); if (ImGui::Button("Cancel", ImVec2(120, 0))) @@ -443,6 +459,176 @@ namespace return s.state == PipelineState::Succeeded ? 0 : 1; } + std::atomic g_interrupts{ 0 }; + + extern "C" void onInterrupt(int) + { + g_interrupts++; + std::signal(SIGINT, onInterrupt); // some C libraries reset the handler after each signal + } + + std::string pointsText(uint64_t points) + { + char buffer[32]; + std::snprintf(buffer, sizeof(buffer), "%.1f M", points / 1e6); + return buffer; + } + + //! `easy_gui --batch `: batch mode without a window, with the GUI's rules: every + //! recording that passes the check is processed, finished ones are skipped. Ctrl+C once stops after the current + //! recording, twice stops now. Exit code 0 = every selected recording succeeded (or was already done), 1 = some + //! failed, were not run or the batch was refused, 2 = usage. + int runBatchHeadless(const fs::path& parent, const fs::path& output_root, const Settings& settings, double check_band_outer_m) + { + std::setvbuf(stdout, nullptr, _IONBF, 0); + std::error_code ec; + if (!fs::is_directory(parent, ec)) + { + std::printf("[X] The folder '%s' does not exist or is not a folder.\n", parent.string().c_str()); + return 1; + } + std::printf("Looking for recordings in '%s' and checking them...\n", parent.string().c_str()); + BatchPlan plan = planBatch(parent, output_root); + BatchRequest& request = plan.request; + request.step1_executable = findStep1Executable(); + request.settings = settings; + request.check_band_outer_m = check_band_outer_m; + for (const auto& f : request.skipped) + std::printf("[skipped] %s: %s\n", f.name.c_str(), f.skip_reason.c_str()); + if (request.items.empty()) + { + std::printf("[X] No recordings found in '%s' or its subfolders.\n", parent.string().c_str()); + return 1; + } + if (const std::string problem = batchOutputProblem(output_root, plan.found); !problem.empty()) + { + std::printf("[X] %s\n", problem.c_str()); + return 1; + } + + for (size_t i = 0; i < request.items.size(); i++) + { + const BatchItem& item = request.items[i]; + const BatchCheck& c = plan.checks[i]; + const std::string label = item.name + (item.output_name == item.name ? "" : " -> " + item.output_name); + if (c.done) + std::printf("[done] %s: already processed (finished %s)\n", label.c_str(), c.done->finished.c_str()); + else if (!item.include) + std::printf("[X] %s: %s\n", label.c_str(), firstError(c.preflight).c_str()); + else + { + std::printf( + "[OK] %s: %s, %s points\n", + label.c_str(), + formatDuration(c.preflight.duration_s).c_str(), + pointsText(c.preflight.total_points).c_str()); + for (const auto& f : c.preflight.findings) + if (f.severity == Severity::Warning) + std::printf(" [!] %s\n", f.message.c_str()); + } + } + if (std::none_of( + request.items.begin(), + request.items.end(), + [](const BatchItem& item) + { + return item.include; + })) + { + std::printf("[X] None of the recordings passed the check; nothing to process.\n"); + return 1; + } + + uint64_t available = 0; + fs::path checked; + const bool known = availableSpace(output_root, available, checked); + const Finding disk = batchDiskFinding(batchDiskNeed(plan.toProcess()), known, available); + std::printf( + "%s %s\n", + disk.severity == Severity::Error ? "[X]" + : disk.severity == Severity::Warning ? "[!]" + : "[OK]", + disk.message.c_str()); + if (disk.severity == Severity::Error) + return 1; + + BatchRunner runner; + std::string error; + if (!runner.start(request, error)) + { + std::printf("[X] %s\n", error.c_str()); + return 1; + } + std::signal(SIGINT, onInterrupt); + int interrupts_seen = 0; + std::string last; + auto last_print = std::chrono::steady_clock::now() - std::chrono::seconds(5); + while (runner.busy()) + { + std::this_thread::sleep_for(std::chrono::milliseconds(500)); + if (const int n = g_interrupts.load(); n != interrupts_seen) + { + interrupts_seen = n; + if (n == 1) + { + runner.stopAfterCurrent(); + std::printf("Stopping after the current recording. Press Ctrl+C again to stop now.\n"); + } + else + { + runner.stopNow(); + std::printf("Stopping now.\n"); + } + } + const BatchStatus s = runner.status(); + std::string line = "overall " + std::to_string(static_cast(s.fraction * 100)) + "%"; + if (s.eta_s >= 0.0) + line += ", about " + formatDuration(s.eta_s) + " left"; + if (s.current >= 0) + { + const PipelineStatus& p = s.current_status; + std::string stage = p.stage == PipelineStage::Odometry ? phaseLabel(p.step1.phase) + : p.stage == PipelineStage::Export ? "Export" + : "Preparing"; + line += " | " + std::to_string(s.current + 1) + "/" + std::to_string(request.items.size()) + " " + + request.items[s.current].name + ": " + stage + " " + std::to_string(static_cast(pipelineFraction(p) * 100)) + "%"; + } + const auto now = std::chrono::steady_clock::now(); + if (line != last && now - last_print >= std::chrono::seconds(5)) + { + std::printf("[%s] %s\n", formatDuration(s.elapsed_s).c_str(), line.c_str()); + last = line; + last_print = now; + } + } + std::signal(SIGINT, SIG_DFL); + + const BatchStatus s = runner.status(); + bool all_good = true; + for (size_t i = 0; i < request.items.size(); i++) + { + const BatchItemResult& r = s.items[i]; + const bool good = r.state == BatchItemState::Succeeded || r.state == BatchItemState::AlreadyDone; + if (request.items[i].include && !good) + all_good = false; + std::printf( + "%s %s: %s%s%s\n", + good ? "[OK]" + : r.state == BatchItemState::Excluded ? "[--]" + : "[X]", + request.items[i].name.c_str(), + batchStateToken(r.state), + r.error.empty() ? "" : ": ", + r.error.c_str()); + } + std::printf( + "Batch %s after %s. Summary: %s\n", + s.stop_after_current ? "stopped" : "finished", + formatDuration(s.elapsed_s).c_str(), + (output_root / BatchRunner::kSummaryFileName).string().c_str()); + return all_good ? 0 : 1; + } + //! Dev aid: `easy_gui --export-only [--check-band ]` runs only the //! export, on an existing step 1 session of that recording (step 1 takes ~20 min on a 10-min walk; the export //! about a minute). The output folder may exist; the three result files in it are overwritten. @@ -538,7 +724,7 @@ namespace int main(int argc, char* argv[]) { std::vector args(argv + 1, argv + argc); - if (!args.empty() && (args[0] == "--headless" || args[0] == "--export-only")) + if (!args.empty() && (args[0] == "--headless" || args[0] == "--export-only" || args[0] == "--batch")) { attachToParentConsole(); const double check_band = takeCheckBand(args); @@ -548,13 +734,20 @@ int main(int argc, char* argv[]) return runHeadless(args[1], args[2], settings, check_band); if (args[0] == "--export-only" && args.size() == 4) return runExportOnly(args[1], args[2], args[3], settings, check_band); + if (args[0] == "--batch" && args.size() == 3) + return runBatchHeadless(args[1], args[2], settings, check_band); std::printf( "usage: %s --headless [--check-band ] [--place-by-raw-time]\n" + " %s --batch [--check-band ] [--place-by-raw-time]\n" " %s --export-only [--check-band ] [--place-by-raw-time]\n" + "--batch: processes every recording in (one folder level down; the parent itself too, if it is a\n" + " recording) into //. Recordings finished in an earlier run are skipped.\n" + " Ctrl+C stops after the current recording, a second Ctrl+C stops now.\n" "--check-band (testing): also export the raw points between the 1.0 m load filter and , flagged\n" " point_origin = 3, to check that near points line up with the odometry points.\n" "--place-by-raw-time: place all points from the raw files at their real time (export option, default off).\n", argv[0], + argv[0], argv[0]); return 2; } diff --git a/apps/easy_gui/tests/test_batch.cpp b/apps/easy_gui/tests/test_batch.cpp index 8cd06168..3938210d 100644 --- a/apps/easy_gui/tests/test_batch.cpp +++ b/apps/easy_gui/tests/test_batch.cpp @@ -148,28 +148,9 @@ namespace }; }; - //! What the GUI and --batch do before starting: discover, name, check, select the recordings that passed. BatchRequest prepareBatch(const fs::path& parent, const fs::path& output_root) { - BatchRequest request; - request.output_root = output_root; - std::vector recordings; - for (const auto& f : discoverRecordings(parent)) - (f.is_recording ? recordings : request.skipped).push_back(f); - const auto names = defaultOutputNames(recordings, output_root); - for (size_t i = 0; i < recordings.size(); i++) - { - const BatchCheck check = checkBatchRecording(recordings[i].path, output_root / names[i]); - BatchItem item{ recordings[i].path, recordings[i].name, names[i] }; - item.total_points = check.preflight.total_points; - if (!check.done && !check.preflight.ok()) - { - item.include = false; - item.exclude_reason = firstError(check.preflight); - } - request.items.push_back(item); - } - return request; + return planBatch(parent, output_root).request; } BatchStatus runToEnd(BatchRunner& runner, const BatchRequest& request) diff --git a/apps/easy_gui/tests/test_pipeline_integration.cpp b/apps/easy_gui/tests/test_pipeline_integration.cpp index 7eb0c513..d9275f49 100644 --- a/apps/easy_gui/tests/test_pipeline_integration.cpp +++ b/apps/easy_gui/tests/test_pipeline_integration.cpp @@ -1,5 +1,6 @@ #include +#include "batch.h" #include "pipeline.h" #include "test_helpers.h" @@ -66,23 +67,31 @@ TEST_CASE("cancel stops a real step 1 run during odometry and cleans up" * docte CHECK(fs::exists(fs::path(testRecording()))); // and the recording, of course } +namespace +{ + //! One file pair of real data, cut to its first 200 points: it passes the pre-flight check, so step 1 really runs, + //! finds no usable data and must fail gracefully. + void writeTooShortRecording(const fs::path& rec) + { + fs::create_directories(rec); + const fs::path source(testRecording()); + fs::copy_file(source / "imu0000.csv", rec / "imu0000.csv"); + fs::copy_file(source / "lidar0000.sn", rec / "lidar0000.sn"); + // Timestamps inside the IMU range, so the pre-flight check passes and step 1 really runs on unusable data. + std::ifstream imu(rec / "imu0000.csv"); + std::string header, first_sample; + std::getline(imu, header); + std::getline(imu, first_sample); + const double imu_t0 = std::stod(first_sample.substr(0, first_sample.find(' '))) / 1e9; + easy_gui_test::writeRawLaz(rec / "lidar0000.laz", 200, imu_t0 + 1.0); + } +} // namespace + TEST_CASE("a recording too short for odometry fails with a message, not a crash" * doctest::skip(testRecording() == nullptr)) { - // One file pair of real data, cut to its first 200 points: step 1 starts, finds no usable data and must fail - // gracefully through Easy GUI. easy_gui_test::TempDir tmp; const fs::path rec = tmp.path / "rec"; - fs::create_directories(rec); - const fs::path source(testRecording()); - fs::copy_file(source / "imu0000.csv", rec / "imu0000.csv"); - fs::copy_file(source / "lidar0000.sn", rec / "lidar0000.sn"); - // Timestamps inside the IMU range, so the pre-flight check passes and step 1 really runs on unusable data. - std::ifstream imu(rec / "imu0000.csv"); - std::string header, first_sample; - std::getline(imu, header); - std::getline(imu, first_sample); - const double imu_t0 = std::stod(first_sample.substr(0, first_sample.find(' '))) / 1e9; - easy_gui_test::writeRawLaz(rec / "lidar0000.laz", 200, imu_t0 + 1.0); + writeTooShortRecording(rec); Pipeline pipeline; std::string error; @@ -126,3 +135,81 @@ TEST_CASE("a real run ends with the point cloud, the trajectory and the README" CHECK(fs::exists(layout.session / "session.mjs")); // kept for step 2 CHECK_FALSE(fs::exists(layout.staging)); } + +TEST_CASE( + "batch: two copies of a real recording and a broken one; the broken one fails, a restart skips the rest" * + doctest::skip(testRecording() == nullptr)) +{ + easy_gui_test::TempDir tmp; + const fs::path parent = tmp.path / "field_day"; + for (const char* copy : { "a_copy", "c_copy" }) + { + fs::create_directories(parent / copy); + for (const auto& entry : fs::directory_iterator(testRecording())) + if (entry.is_regular_file()) + fs::create_symlink(fs::canonical(entry.path()), parent / copy / entry.path().filename()); + } + writeTooShortRecording(parent / "b_broken"); + fs::create_directories(parent / "notes"); + const fs::path out = tmp.path / "results"; + + BatchPlan plan = planBatch(parent, out); + REQUIRE(plan.request.items.size() == 3); + REQUIRE(plan.request.skipped.size() == 1); + for (const auto& item : plan.request.items) + REQUIRE_MESSAGE(item.include, item.exclude_reason); + plan.request.step1_executable = findStep1Executable(); + + { + BatchRunner runner; + std::string error; + REQUIRE(runner.start(plan.request, error)); + runner.wait(); + const BatchStatus s = runner.status(); + CHECK_MESSAGE(s.items[0].state == BatchItemState::Succeeded, s.items[0].error); + CHECK(s.items[1].state == BatchItemState::Failed); + CHECK_FALSE(s.items[1].error.empty()); + CHECK_MESSAGE(s.items[2].state == BatchItemState::Succeeded, s.items[2].error); + MESSAGE("broken recording: " << s.items[1].error); + for (const char* copy : { "a_copy", "c_copy" }) + { + const OutputLayout layout(out / copy); + CHECK(fs::file_size(layout.point_cloud) > 0); + CHECK(fs::exists(layout.trajectory)); + CHECK(fs::exists(layout.done_marker)); + CHECK_FALSE(fs::exists(layout.staging)); + } + CHECK(s.items[0].exported_points == s.items[2].exported_points); // step 1 is deterministic + CHECK(fs::exists(out / "b_broken" / "easy_gui.log")); + CHECK_FALSE(fs::exists(OutputLayout(out / "b_broken").done_marker)); + CHECK(fs::exists(out / BatchRunner::kLogFileName)); + } + + // Restart: the copies are skipped, the broken one is retried (and fails again). + BatchPlan again = planBatch(parent, out); + REQUIRE(again.request.items.size() == 3); + CHECK(again.checks[0].done.has_value()); + CHECK_FALSE(again.checks[1].done.has_value()); + CHECK(again.request.items[1].include); // its unfinished attempt (the log) does not block it + again.request.step1_executable = findStep1Executable(); + BatchRunner runner; + std::string error; + REQUIRE(runner.start(again.request, error)); + runner.wait(); + const BatchStatus s = runner.status(); + CHECK(s.items[0].state == BatchItemState::AlreadyDone); + CHECK(s.items[1].state == BatchItemState::Failed); + CHECK(s.items[2].state == BatchItemState::AlreadyDone); + + std::ifstream csv(out / BatchRunner::kSummaryFileName); + std::string header, a, b, c, notes; + std::getline(csv, header); + std::getline(csv, a); + std::getline(csv, b); + std::getline(csv, c); + std::getline(csv, notes); + CHECK(a.find(",a_copy,already_done,") != std::string::npos); + CHECK(b.find(",b_broken,failed,") != std::string::npos); + CHECK(c.find(",c_copy,already_done,") != std::string::npos); + CHECK(notes.find(",,skipped,") != std::string::npos); +} From b3269fd58b5f67730adc52755c434b7324a46658 Mon Sep 17 00:00:00 2001 From: Martin Mokros Date: Sun, 18 Oct 2026 15:13:11 +0100 Subject: [PATCH 21/23] easy_gui: "Several recordings" mode in the window A toggle between one recording and several. In batch mode the user picks a parent folder; the table lists each recording with an include checkbox, its length, the pre-flight result (checked row by row in the background) and an editable output folder name (validated, case-insensitive collisions flagged). Folders that are not recordings are listed as skipped with the reason. Below the table: each problem in words and the batch disk check. While running: overall progress with ETA, the current recording's stage, per-row status, "Stop after this recording" and "Stop now". The default output folder is reused when it holds an earlier batch, so starting the same batch again continues it. The main window now gets a scrollbar when its content is taller than the window. --screenshot accepts --batch ; EASY_GUI_SCREENSHOT_START and EASY_GUI_SCREENSHOT_FRAME capture a running batch for documentation. Co-Authored-By: Claude Opus 5.5 --- apps/easy_gui/CMakeLists.txt | 4 +- apps/easy_gui/src/easy_gui_main.cpp | 766 +++++++++++++++++++++++++++- 2 files changed, 758 insertions(+), 12 deletions(-) diff --git a/apps/easy_gui/CMakeLists.txt b/apps/easy_gui/CMakeLists.txt index 7e0f5720..5bf9b474 100644 --- a/apps/easy_gui/CMakeLists.txt +++ b/apps/easy_gui/CMakeLists.txt @@ -84,7 +84,9 @@ if(MSVC) endif() # The window. raylib + rlImGui like step 2; core_pfd for the native folder picker. -add_executable(easy_gui src/easy_gui_main.cpp) +# imgui_stdlib (std::string InputText, for the batch table's output names) is compiled here: imgui_raylib does not +# include it, and it must come from the same vendored ImGui. +add_executable(easy_gui src/easy_gui_main.cpp ${THIRDPARTY_DIRECTORY}/imgui/misc/cpp/imgui_stdlib.cpp) target_link_libraries(easy_gui PRIVATE easy_gui_core diff --git a/apps/easy_gui/src/easy_gui_main.cpp b/apps/easy_gui/src/easy_gui_main.cpp index 31afff27..bacf75f9 100644 --- a/apps/easy_gui/src/easy_gui_main.cpp +++ b/apps/easy_gui/src/easy_gui_main.cpp @@ -13,15 +13,19 @@ #include #include +#include #include #include #include #include +#include #include #include #include +#include #include +#include #include #include #include @@ -78,8 +82,318 @@ namespace ImGui::PopStyleColor(); } + std::string folderName(const fs::path& folder) + { + const fs::path normal = folder.lexically_normal(); + return normal.filename().empty() ? normal.parent_path().filename().string() : normal.filename().string(); + } + + //! ~/Documents/Easy Forest HDMapping/. Unlike single mode, an existing folder from an earlier + //! batch is reused (restarting the same batch continues it); _2, _3, ... only when the folder holds something else. + fs::path defaultBatchOutputFolder(const fs::path& parent) + { + const fs::path base = homeFolder() / "Documents" / "Easy Forest HDMapping"; + const std::string name = folderName(parent); + std::error_code ec; + fs::path candidate = base / name; + for (int i = 2; + fs::exists(candidate, ec) && !fs::is_empty(candidate, ec) && !fs::exists(candidate / BatchRunner::kSummaryFileName, ec); + i++) + candidate = base / (name + "_" + std::to_string(i)); + return candidate; + } + + //! Checks the batch table's rows one at a time on a worker thread: they read the recordings, which may be on a + //! slow USB stick, and the window must stay responsive. Each result carries the generation it was requested for, + //! so results for a parent folder the user has since replaced are dropped. + class RowChecker + { + public: + struct Result + { + size_t row; + int generation; + fs::path output_folder; // the check was made against this output folder + BatchCheck check; + }; + + RowChecker() + { + thread_ = std::thread(&RowChecker::run, this); + } + ~RowChecker() + { + { + std::lock_guard lock(mutex_); + stop_ = true; + } + cv_.notify_all(); + thread_.join(); + } + + void request(size_t row, int generation, const fs::path& recording, const fs::path& output_folder) + { + std::lock_guard lock(mutex_); + std::erase_if( + queue_, + [&](const Job& j) + { + return j.generation != generation || j.row == row; + }); + queue_.push_back({ row, generation, recording, output_folder }); + cv_.notify_all(); + } + + std::vector take() + { + std::lock_guard lock(mutex_); + return std::exchange(results_, {}); + } + + private: + struct Job + { + size_t row; + int generation; + fs::path recording; + fs::path output_folder; + }; + + void run() + { + std::unique_lock lock(mutex_); + while (true) + { + cv_.wait( + lock, + [&] + { + return stop_ || !queue_.empty(); + }); + if (stop_) + return; + const Job job = queue_.front(); + queue_.pop_front(); + lock.unlock(); + BatchCheck check = checkBatchRecording(job.recording, job.output_folder); + lock.lock(); + results_.push_back({ job.row, job.generation, job.output_folder, std::move(check) }); + } + } + + std::mutex mutex_; + std::condition_variable cv_; + std::deque queue_; + std::vector results_; + bool stop_ = false; + std::thread thread_; // last: starts after the members above exist + }; + + struct BatchRow + { + DiscoveredFolder folder; + std::string output_name; + bool name_edited = false; // keep the user's name when the output folder changes + bool selected = true; // the user's choice; a recording that fails the check is never included + std::optional check; + fs::path checked_output; + + bool checkCurrent(const fs::path& output_root) const + { + return check && checked_output == output_root / output_name; + } + bool usable() const + { + return check && (check->done || check->preflight.ok()); + } + bool included() const + { + return selected && usable(); + } + //! Will write into its output folder, as far as is known: included, or selected and not checked yet. + bool active(const fs::path& output_root) const + { + return selected && (!checkCurrent(output_root) || usable()); + } + }; + + struct BatchUi + { + fs::path parent; + fs::path output_root; + bool output_chosen_by_user = false; + std::vector found; + std::vector rows; // the recordings + std::vector skipped; // the other folders + int generation = 0; + RowChecker checker; + + bool disk_known = false; + uint64_t available = 0; + double disk_checked_at = -1e9; + + BatchRunner runner; + bool started = false; // the runner was started; the table shows the status column until "new batch" + std::string start_error; + + void setParent(const fs::path& folder) + { + parent = folder; + found = discoverRecordings(folder); + rows.clear(); + skipped.clear(); + std::vector recordings; + for (const auto& f : found) + (f.is_recording ? recordings : skipped).push_back(f); + if (!output_chosen_by_user) + output_root = defaultBatchOutputFolder(folder); + const auto names = defaultOutputNames(recordings, output_root); + for (size_t i = 0; i < recordings.size(); i++) + rows.push_back({ recordings[i], names[i] }); + generation++; + checkAll(); + } + + void setOutputRoot(const fs::path& folder) + { + output_root = folder; + output_chosen_by_user = true; + std::vector recordings; + for (const auto& r : rows) + recordings.push_back(r.folder); + const auto names = defaultOutputNames(recordings, output_root); + for (size_t i = 0; i < rows.size(); i++) + if (!rows[i].name_edited) + rows[i].output_name = names[i]; + checkAll(); + } + + void checkRow(size_t i) + { + if (outputNameProblem(rows[i].output_name).empty()) + checker.request(i, generation, rows[i].folder.path, output_root / rows[i].output_name); + } + + void checkAll() + { + for (size_t i = 0; i < rows.size(); i++) + checkRow(i); + disk_checked_at = -1e9; + } + + void poll() + { + for (auto& r : checker.take()) + if (r.generation == generation && r.row < rows.size()) + { + rows[r.row].check = std::move(r.check); + rows[r.row].checked_output = r.output_folder; + disk_checked_at = -1e9; + } + // Free space changes while other programs run; refreshed every few seconds before the start. + if (!runner.busy() && !started && ImGui::GetTime() - disk_checked_at > 3.0 && !output_root.empty()) + { + fs::path checked; + disk_known = availableSpace(output_root, available, checked); + disk_checked_at = ImGui::GetTime(); + } + } + + bool checking() const + { + for (const auto& r : rows) + if (!r.checkCurrent(output_root) && outputNameProblem(r.output_name).empty()) + return true; + return false; + } + + //! Why row `i`'s output name cannot be used, or empty. + std::string nameProblem(size_t i) const + { + if (std::string p = outputNameProblem(rows[i].output_name); !p.empty()) + return p; + for (size_t j = 0; j < rows.size(); j++) + if (j != i && rows[j].active(output_root) && sameOutputName(rows[i].output_name, rows[j].output_name)) + return "Another recording uses the same output folder name."; + return {}; + } + + std::vector toProcess() const + { + std::vector out; + for (const auto& r : rows) + if (r.included() && r.checkCurrent(output_root) && !r.check->done) + out.push_back(&r.check->preflight); + return out; + } + + Finding diskFinding() const + { + return batchDiskFinding(batchDiskNeed(toProcess()), disk_known, available); + } + + bool readyToStart() const + { + if (runner.busy() || started || rows.empty() || checking() || !batchOutputProblem(output_root, found).empty()) + return false; + bool any = false; + for (size_t i = 0; i < rows.size(); i++) + { + if (!rows[i].active(output_root)) + continue; + if (!nameProblem(i).empty()) + return false; + any = any || rows[i].included(); + } + return any && diskFinding().severity != Severity::Error; + } + + BatchRequest makeRequest(const Settings& settings) const + { + BatchRequest request; + request.output_root = output_root; + request.skipped = skipped; + request.step1_executable = findStep1Executable(); + request.settings = settings; + for (const auto& r : rows) + { + BatchItem item{ r.folder.path, r.folder.name, r.output_name, r.included() }; + item.total_points = r.check ? r.check->preflight.total_points : 0; + if (!r.selected) + item.exclude_reason = "Not selected."; + else if (!r.usable()) + item.exclude_reason = "Check failed: " + firstError(r.check->preflight); + request.items.push_back(std::move(item)); + } + return request; + } + + void reset() + { + runner.wait(); + parent.clear(); + found.clear(); + rows.clear(); + skipped.clear(); + generation++; + if (!output_chosen_by_user) + output_root.clear(); + started = false; + start_error.clear(); + } + }; + + enum class Mode + { + Single, + Batch + }; + struct App { + Mode mode = Mode::Single; + BatchUi batch; + fs::path recording; fs::path output_folder; bool output_chosen_by_user = false; @@ -401,6 +715,398 @@ namespace ImGui::SetScrollHereY(1.0f); ImGui::EndChild(); } + + // --------------------------------------------------------------------------------------------------------------- + // Batch mode ("Several recordings") + + void drawBatchInputs(App& app, bool locked) + { + BatchUi& b = app.batch; + ImGui::BeginDisabled(locked); + + ImGui::SeparatorText("1. Folder with recordings"); + ImGui::TextWrapped( + "A folder that contains several recording folders, for example a copy of the scanner's USB stick. Recordings are looked " + "for one folder level down. You can also drag the folder onto this window."); + if (ImGui::Button("Choose folder...")) + { + const std::string folder = mandeye::fd::SelectFolder("Choose the folder that contains the recordings"); + if (!folder.empty()) + b.setParent(folder); + } + ImGui::SameLine(); + ImGui::TextUnformatted(b.parent.empty() ? "(none chosen)" : b.parent.string().c_str()); + + ImGui::SeparatorText("2. Output folder"); + ImGui::TextWrapped( + "Each recording's results go into a folder of its own in here, named in the table below. Choosing the same output folder " + "again continues an interrupted batch: finished recordings are skipped."); + if (ImGui::Button("Choose output folder...")) + { + const std::string folder = mandeye::fd::SelectFolder("Choose where to save the results"); + if (!folder.empty()) + b.setOutputRoot(folder); + } + ImGui::SameLine(); + ImGui::TextUnformatted(b.output_root.empty() ? "(set automatically once a folder is chosen)" : b.output_root.string().c_str()); + + drawAdvancedSettings(app.settings); + ImGui::EndDisabled(); + } + + void drawCheckCell(const BatchRow& row, bool current) + { + if (!current) + { + ImGui::TextDisabled(outputNameProblem(row.output_name).empty() ? "checking..." : "-"); + return; + } + const BatchCheck& c = *row.check; + if (c.done) + { + ImGui::TextColored(kGreen, "already done"); + if (ImGui::IsItemHovered()) + ImGui::SetTooltip("Finished on %s. It will be skipped.", c.done->finished.c_str()); + return; + } + size_t warnings = 0; + for (const auto& f : c.preflight.findings) + warnings += f.severity == Severity::Warning ? 1 : 0; + if (!c.preflight.ok()) + ImGui::TextColored(kRed, "[X] problem"); + else if (warnings > 0) + ImGui::TextColored(kAmber, "[!] %zu note%s", warnings, warnings > 1 ? "s" : ""); + else + ImGui::TextColored(kGreen, "[OK]"); + if (ImGui::IsItemHovered()) + { + ImGui::BeginTooltip(); + ImGui::PushTextWrapPos(ImGui::GetFontSize() * 40.0f); + for (const auto& f : c.preflight.findings) + { + const char* tag = f.severity == Severity::Error ? "[X]" : f.severity == Severity::Warning ? "[!]" : "[OK]"; + ImGui::TextWrapped("%s %s", tag, f.message.c_str()); + } + ImGui::PopTextWrapPos(); + ImGui::EndTooltip(); + } + } + + void drawStatusCell(const BatchItemResult& r, bool current, const BatchStatus& s) + { + switch (r.state) + { + case BatchItemState::Excluded: + ImGui::TextDisabled("not selected"); + break; + case BatchItemState::Pending: + ImGui::TextDisabled("waiting"); + break; + case BatchItemState::Running: + ImGui::TextColored(kAmber, "running %d%%", current ? static_cast(pipelineFraction(s.current_status) * 100) : 0); + break; + case BatchItemState::Succeeded: + ImGui::TextColored(kGreen, "done (%s)", formatDuration(r.duration_s).c_str()); + break; + case BatchItemState::AlreadyDone: + ImGui::TextColored(kGreen, "already done"); + break; + case BatchItemState::Failed: + ImGui::TextColored(kRed, "failed"); + break; + case BatchItemState::Cancelled: + ImGui::TextColored(kAmber, "stopped"); + break; + case BatchItemState::NotRun: + ImGui::TextDisabled("not run"); + break; + } + if (!r.error.empty() && ImGui::IsItemHovered()) + { + ImGui::BeginTooltip(); + ImGui::PushTextWrapPos(ImGui::GetFontSize() * 40.0f); + ImGui::TextWrapped("%s", r.error.c_str()); + ImGui::PopTextWrapPos(); + ImGui::EndTooltip(); + } + } + + void drawBatchTable(App& app, bool locked) + { + BatchUi& b = app.batch; + ImGui::SeparatorText("3. Recordings"); + if (b.parent.empty()) + { + ImGui::TextDisabled("Choose a folder first."); + return; + } + if (b.rows.empty()) + { + wrappedColored( + kRed, + "No recordings were found in this folder or in the folders directly inside it. A recording folder contains " + "lidar*.laz and imu*.csv files."); + } + + const BatchStatus s = b.runner.status(); + const bool show_status = b.started && s.items.size() == b.rows.size(); + if (!b.rows.empty()) + { + const int columns = show_status ? 6 : 5; + const float row_height = ImGui::GetFrameHeightWithSpacing(); + const float height = row_height * static_cast(std::min(b.rows.size(), 10) + 1) + 6.0f; + if (ImGui::BeginTable( + "recordings", + columns, + ImGuiTableFlags_Borders | ImGuiTableFlags_RowBg | ImGuiTableFlags_ScrollY | ImGuiTableFlags_SizingStretchProp, + ImVec2(0, height))) + { + ImGui::TableSetupScrollFreeze(0, 1); + ImGui::TableSetupColumn("Include", ImGuiTableColumnFlags_WidthFixed); + ImGui::TableSetupColumn("Recording", ImGuiTableColumnFlags_WidthStretch, 3.0f); + ImGui::TableSetupColumn("Length", ImGuiTableColumnFlags_WidthFixed); + ImGui::TableSetupColumn("Check", ImGuiTableColumnFlags_WidthFixed); + ImGui::TableSetupColumn("Output folder name", ImGuiTableColumnFlags_WidthStretch, 3.0f); + if (show_status) + ImGui::TableSetupColumn("Status", ImGuiTableColumnFlags_WidthFixed); + ImGui::TableHeadersRow(); + + for (size_t i = 0; i < b.rows.size(); i++) + { + BatchRow& row = b.rows[i]; + const bool current = row.checkCurrent(b.output_root); + ImGui::PushID(static_cast(i)); + ImGui::TableNextRow(); + + ImGui::TableNextColumn(); + ImGui::BeginDisabled(locked || (current && !row.usable())); + bool include = current ? row.included() : row.selected; + if (ImGui::Checkbox("##include", &include)) + row.selected = include; + ImGui::EndDisabled(); + + ImGui::TableNextColumn(); + ImGui::AlignTextToFramePadding(); + ImGui::TextUnformatted(row.folder.name.c_str()); + if (ImGui::IsItemHovered()) + ImGui::SetTooltip("%s", row.folder.path.string().c_str()); + + ImGui::TableNextColumn(); + if (row.check) + ImGui::TextUnformatted(formatDuration(row.check->preflight.duration_s).c_str()); + else + ImGui::TextDisabled("..."); + + ImGui::TableNextColumn(); + drawCheckCell(row, current); + + ImGui::TableNextColumn(); + const std::string problem = row.active(b.output_root) ? b.nameProblem(i) : std::string(); + ImGui::BeginDisabled(locked); + if (!problem.empty()) + ImGui::PushStyleColor(ImGuiCol_FrameBg, ImVec4(0.45f, 0.12f, 0.12f, 1.0f)); + ImGui::SetNextItemWidth(-FLT_MIN); + ImGui::InputText("##output_name", &row.output_name); + if (ImGui::IsItemDeactivatedAfterEdit()) + { + row.name_edited = true; + b.checkRow(i); + } + if (!problem.empty()) + { + ImGui::PopStyleColor(); + if (ImGui::IsItemHovered()) + ImGui::SetTooltip("%s", problem.c_str()); + } + ImGui::EndDisabled(); + + if (show_status) + { + ImGui::TableNextColumn(); + drawStatusCell(s.items[i], s.current == static_cast(i), s); + } + ImGui::PopID(); + } + ImGui::EndTable(); + } + } + + if (!b.started) + { + // Problems in words, below the table (the table only has room for a mark). + for (size_t i = 0; i < b.rows.size(); i++) + { + const BatchRow& row = b.rows[i]; + if (const std::string problem = row.active(b.output_root) ? b.nameProblem(i) : std::string(); !problem.empty()) + wrappedColored(kRed, row.folder.name + ": output folder name: " + problem); + if (!row.checkCurrent(b.output_root) || row.check->done) + continue; + for (const auto& f : row.check->preflight.findings) + if (f.severity != Severity::Ok) + wrappedColored(f.severity == Severity::Error ? kRed : kAmber, row.folder.name + ": " + f.message); + } + if (const std::string problem = batchOutputProblem(b.output_root, b.found); !problem.empty()) + wrappedColored(kRed, problem); + else if (!b.rows.empty() && !b.checking()) + { + const Finding disk = b.diskFinding(); + wrappedColored( + disk.severity == Severity::Error ? kRed + : disk.severity == Severity::Warning ? kAmber + : kGreen, + disk.message); + } + } + + if (!b.skipped.empty() && + ImGui::TreeNode("skipped", "%zu folder%s skipped (not recordings)", b.skipped.size(), b.skipped.size() > 1 ? "s" : "")) + { + for (const auto& f : b.skipped) + ImGui::BulletText("%s: %s", f.name.c_str(), f.skip_reason.c_str()); + ImGui::TreePop(); + } + } + + void drawBatchRun(App& app) + { + BatchUi& b = app.batch; + ImGui::SeparatorText("4. Process"); + if (!b.started) + { + size_t selected = 0; + for (const auto& r : b.rows) + selected += r.included() ? 1 : 0; + ImGui::BeginDisabled(!b.readyToStart()); + const std::string label = "Start processing " + std::to_string(selected) + " recording" + (selected == 1 ? "" : "s"); + if (ImGui::Button(label.c_str(), ImVec2(260, 0))) + { + b.start_error.clear(); + if (b.runner.start(b.makeRequest(app.settings), b.start_error)) + b.started = true; + } + ImGui::EndDisabled(); + if (!b.start_error.empty()) + wrappedColored(kRed, b.start_error); + return; + } + + const BatchStatus s = b.runner.status(); + const BatchRequest& request = b.runner.request(); + size_t selected = 0, finished = 0; + size_t counts[8] = {}; + for (size_t i = 0; i < s.items.size(); i++) + { + counts[static_cast(s.items[i].state)]++; + if (!request.items[i].include) + continue; + selected++; + if (s.items[i].state != BatchItemState::Pending && s.items[i].state != BatchItemState::Running) + finished++; + } + + if (s.running) + { + std::string overall = std::to_string(finished) + " of " + std::to_string(selected) + " recordings finished, elapsed " + + formatDuration(s.elapsed_s); + if (s.eta_s >= 0.0) + overall += ", about " + formatDuration(s.eta_s) + " left"; + ImGui::TextUnformatted("All recordings"); + ImGui::ProgressBar(static_cast(s.fraction), ImVec2(-1, 0), overall.c_str()); + ImGui::Spacing(); + if (s.current >= 0) + { + ImGui::Text( + "Now: %s (%d%% of this recording)", + request.items[s.current].name.c_str(), + static_cast(pipelineFraction(s.current_status) * 100)); + drawStageProgress(s.current_status); + } + else + ImGui::TextUnformatted("Checking the next recording..."); + + ImGui::Spacing(); + ImGui::BeginDisabled(s.stop_after_current); + if (ImGui::Button(s.stop_after_current ? "Stopping after this recording..." : "Stop after this recording", ImVec2(260, 0))) + b.runner.stopAfterCurrent(); + ImGui::EndDisabled(); + ImGui::SameLine(); + if (ImGui::Button("Stop now", ImVec2(120, 0))) + ImGui::OpenPopup("Stop now?"); + if (ImGui::BeginPopupModal("Stop now?", nullptr, ImGuiWindowFlags_AlwaysAutoResize)) + { + ImGui::TextUnformatted( + "Stop now? The recording being processed is discarded.\nRecordings that are already finished are kept."); + if (ImGui::Button("Yes, stop now", ImVec2(140, 0))) + { + b.runner.stopNow(); + ImGui::CloseCurrentPopup(); + } + ImGui::SameLine(); + if (ImGui::Button("No, continue", ImVec2(140, 0))) + ImGui::CloseCurrentPopup(); + ImGui::EndPopup(); + } + ImGui::TextDisabled("The computer is kept from going to sleep while the batch runs."); + return; + } + + const size_t failed = counts[static_cast(BatchItemState::Failed)] + counts[static_cast(BatchItemState::Cancelled)]; + const size_t not_run = counts[static_cast(BatchItemState::NotRun)]; + std::string summary = std::string(s.stop_after_current ? "Batch stopped" : "Batch finished") + " after " + + formatDuration(s.elapsed_s) + ": " + std::to_string(counts[static_cast(BatchItemState::Succeeded)]) + " succeeded"; + if (counts[static_cast(BatchItemState::AlreadyDone)]) + summary += ", " + std::to_string(counts[static_cast(BatchItemState::AlreadyDone)]) + " already done before"; + if (failed) + summary += ", " + std::to_string(failed) + " failed or stopped"; + if (not_run) + summary += ", " + std::to_string(not_run) + " not run"; + summary += "."; + wrappedColored(failed || not_run ? kAmber : kGreen, summary); + if (failed) + ImGui::TextWrapped( + "Point at 'failed' in the table to see why. Each recording's own easy_gui.log has the details. Start the same batch " + "again to retry the recordings that did not finish."); + ImGui::TextDisabled("batch_summary.csv and batch.log are in %s", request.output_root.string().c_str()); + + if (ImGui::Button("Open output folder")) + OpenURL(request.output_root.string().c_str()); + ImGui::SameLine(); + if (ImGui::Button("Process another batch")) + b.reset(); + } + + void drawLogLines(const char* id, const std::vector& lines) + { + ImGui::BeginChild(id, ImVec2(0, 200), ImGuiChildFlags_Borders, ImGuiWindowFlags_HorizontalScrollbar); + for (const auto& line : lines) + ImGui::TextUnformatted(line.c_str()); + if (ImGui::GetScrollY() >= ImGui::GetScrollMaxY() - 5.0f) // follow the end unless the user scrolled up + ImGui::SetScrollHereY(1.0f); + ImGui::EndChild(); + } + + void drawBatchLog(App& app) + { + ImGui::Spacing(); + if (!ImGui::CollapsingHeader("Details (processing log)")) + return; + ImGui::TextUnformatted("Batch (batch.log)"); + drawLogLines("batch_log", app.batch.runner.logTail()); + ImGui::TextUnformatted("Current recording (its easy_gui.log)"); + drawLogLines("recording_log", app.batch.runner.currentLogTail()); + } + + void drawModeSwitch(App& app, bool locked) + { + ImGui::BeginDisabled(locked); + int mode = static_cast(app.mode); + ImGui::RadioButton("One recording", &mode, static_cast(Mode::Single)); + ImGui::SameLine(); + ImGui::RadioButton("Several recordings", &mode, static_cast(Mode::Batch)); + app.mode = static_cast(mode); + ImGui::EndDisabled(); + ImGui::Spacing(); + } } // namespace namespace @@ -752,8 +1458,8 @@ int main(int argc, char* argv[]) return 2; } - // Dev aid: `--screenshot [recording]` renders ~2 s, saves the window and exits (layout checks without - // capturing the whole desktop). + // Dev aid: `--screenshot [recording | --batch ]` renders ~2 s, saves the window and exits (layout checks + // without capturing the whole desktop). std::string screenshot_file; int first_positional = 1; if (argc > 2 && std::string(argv[1]) == "--screenshot") @@ -763,7 +1469,12 @@ int main(int argc, char* argv[]) } App app; - if (argc > first_positional && fs::is_directory(argv[first_positional])) + if (argc > first_positional + 1 && std::string(argv[first_positional]) == "--batch" && fs::is_directory(argv[first_positional + 1])) + { + app.mode = Mode::Batch; + app.batch.setParent(argv[first_positional + 1]); + } + else if (argc > first_positional && fs::is_directory(argv[first_positional])) app.setRecording(argv[first_positional]); SetConfigFlags(FLAG_WINDOW_RESIZABLE | FLAG_MSAA_4X_HINT); @@ -778,7 +1489,10 @@ int main(int argc, char* argv[]) while (!WindowShouldClose()) { app.poll(); - const bool locked = app.pipeline.busy() || app.pipeline.status().state != PipelineState::Idle; + app.batch.poll(); + const bool single_locked = app.pipeline.busy() || app.pipeline.status().state != PipelineState::Idle; + const bool batch_locked = app.batch.runner.busy() || app.batch.started; + const bool locked = app.mode == Mode::Single ? single_locked : batch_locked; if (IsFileDropped()) { @@ -787,7 +1501,11 @@ int main(int argc, char* argv[]) { fs::path p(dropped.paths[0]); std::error_code ec; - app.setRecording(fs::is_directory(p, ec) ? p : p.parent_path()); + const fs::path folder = fs::is_directory(p, ec) ? p : p.parent_path(); + if (app.mode == Mode::Single) + app.setRecording(folder); + else + app.batch.setParent(folder); } UnloadDroppedFiles(dropped); } @@ -800,22 +1518,47 @@ int main(int argc, char* argv[]) ImGui::SetNextWindowPos(viewport->WorkPos); ImGui::SetNextWindowSize(viewport->WorkSize); ImGui::Begin( - "Easy Forest HDMapping", nullptr, ImGuiWindowFlags_NoDecoration | ImGuiWindowFlags_NoMove | ImGuiWindowFlags_NoSavedSettings); + "Easy Forest HDMapping", + nullptr, + // A scrollbar appears when the content is taller than the window (batch table, open log). + ImGuiWindowFlags_NoTitleBar | ImGuiWindowFlags_NoResize | ImGuiWindowFlags_NoCollapse | ImGuiWindowFlags_NoMove | + ImGuiWindowFlags_NoSavedSettings); ImGui::TextUnformatted("Easy Forest HDMapping"); ImGui::TextDisabled("Mandeye recording -> lidar odometry (precision forestry settings) -> point cloud and trajectory"); ImGui::Spacing(); + drawModeSwitch(app, single_locked || batch_locked); - drawInputs(app, locked); - drawPreflight(app); - drawRun(app); - drawLog(app); + if (app.mode == Mode::Single) + { + drawInputs(app, locked); + drawPreflight(app); + drawRun(app); + drawLog(app); + } + else + { + drawBatchInputs(app, locked); + drawBatchTable(app, locked); + drawBatchRun(app); + drawBatchLog(app); + } ImGui::End(); rlImGuiEnd(); EndDrawing(); - if (!screenshot_file.empty() && ++frame == 60) + // Dev aid for documentation screenshots of a running batch: EASY_GUI_SCREENSHOT_START=1 starts it once ready, + // EASY_GUI_SCREENSHOT_FRAME= takes the screenshot at frame n (30 frames per second) instead of 60. + static const bool auto_start = !screenshot_file.empty() && std::getenv("EASY_GUI_SCREENSHOT_START"); + if (auto_start && app.mode == Mode::Batch && app.batch.readyToStart()) + { + if (app.batch.runner.start(app.batch.makeRequest(app.settings), app.batch.start_error)) + app.batch.started = true; + } + static const int screenshot_frame = + std::getenv("EASY_GUI_SCREENSHOT_FRAME") ? std::max(1, std::atoi(std::getenv("EASY_GUI_SCREENSHOT_FRAME"))) : 60; + if (!screenshot_file.empty() && ++frame == screenshot_frame) { TakeScreenshot(screenshot_file.c_str()); break; @@ -824,6 +1567,7 @@ int main(int argc, char* argv[]) // Closing the window while processing stops the child process (Pipeline's destructor cancels and joins). app.pipeline.cancel(); + app.batch.runner.stopNow(); rlImGuiShutdown(); CloseWindow(); return 0; From eb35b7b4f0ee1182bf6ea8ee05fc560ee407ab69 Mon Sep 17 00:00:00 2001 From: Martin Mokros Date: Sun, 18 Oct 2026 15:18:12 +0100 Subject: [PATCH 22/23] Easy Forest HDMapping 0.2.0: batch mode docs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit README: new section "Processing many recordings" (the table, stopping and continuing a batch, batch_summary.csv and batch.log, --batch), troubleshooting entries, and the scope note now says one point cloud per walk. DESIGN.md: new ยง10 Batch mode (discovery, output names, checks, done markers and restarting, sleep prevention, progress, tests with measured numbers); status M4. EASY_FOREST_VERSION 0.2.0 (not released yet). Co-Authored-By: Claude Opus 5.5 --- apps/easy_gui/CMakeLists.txt | 2 +- apps/easy_gui/README.md | 69 +++++++++++++++++- apps/easy_gui/docs/DESIGN.md | 133 +++++++++++++++++++++++++++++++++-- apps/easy_gui/src/version.h | 2 +- 4 files changed, 195 insertions(+), 11 deletions(-) diff --git a/apps/easy_gui/CMakeLists.txt b/apps/easy_gui/CMakeLists.txt index 5bf9b474..885e33c1 100644 --- a/apps/easy_gui/CMakeLists.txt +++ b/apps/easy_gui/CMakeLists.txt @@ -4,7 +4,7 @@ project(easy_gui) # Easy Forest HDMapping's own version: shown next to HDMapping's in the window title, README.txt, easy_gui.log and # the LAZ header. Keep it equal to the easy-forest-v* release tag. -set(EASY_FOREST_VERSION "0.1.0") +set(EASY_FOREST_VERSION "0.2.0") # easy_gui_core -- everything except the window: pre-flight check, staging, the # step 1 child process and its progress parsing, and the pipeline that ties diff --git a/apps/easy_gui/README.md b/apps/easy_gui/README.md index 81a24dd7..11faeffa 100644 --- a/apps/easy_gui/README.md +++ b/apps/easy_gui/README.md @@ -10,6 +10,7 @@ is nothing to tune. - [What your computer needs](#what-your-computer-needs) - [Installation](#installation) - [How to use it](#how-to-use-it) +- [Processing many recordings](#processing-many-recordings) - [What you get](#what-you-get) - [Troubleshooting](#troubleshooting) - [Getting help](#getting-help) @@ -23,7 +24,8 @@ its settings are tuned for forests. What it does **not** do: -- **One walk at a time.** It processes one recording folder. It does not merge several walks into one cloud. +- **One point cloud per walk.** Each recording becomes its own point cloud. It can process many recordings in one go + (see [Processing many recordings](#processing-many-recordings)), but it does not merge several walks into one cloud. - **No real-world coordinates.** It does not use GNSS or ground control points. Coordinates are in metres, starting at the spot where you started walking, with the vertical axis (Z) pointing up. - **No loop closure yet.** If you return to your starting point, small drift is not corrected. (Planned for a later @@ -94,7 +96,8 @@ from your desktop next time. ## How to use it -The window guides you through four steps, top to bottom. +At the top of the window, choose **One recording** (described here) or **Several recordings** (see +[Processing many recordings](#processing-many-recordings)). The window guides you through four steps, top to bottom. **1. Recording.** Click **Choose recordingโ€ฆ** and select the folder the scanner wrote, for example `continousScanning_0012`. It contains files named `lidar0000.laz`, `imu0000.csv` and so on. You can also drag the @@ -124,6 +127,64 @@ When it is done, click **Open output folder** to see the results, or **Process a it slightly less sharp than the default, so leave it off unless you are comparing the two. - **Keep intermediate files**: keeps extra working files, useful when reporting a problem. +## Processing many recordings + +After a field day you often have many recordings, for example one per plot. Easy Forest HDMapping can process all of +them one after another, unattended, for example overnight. + +1. At the top of the window, choose **Several recordings**. +2. **Folder with recordings.** Click **Choose folderโ€ฆ** and select the folder that *contains* the recording folders, + for example a copy of the scanner's USB stick. You can also drag it onto the window. Easy Forest HDMapping looks in + the folders directly inside it (not deeper). If the folder you chose is itself a recording, it is included too. +3. **Output folder.** Easy Forest HDMapping suggests `Documents/Easy Forest HDMapping/`. Each + recording's results go into a folder of its own inside it. +4. **Recordings.** A table lists every recording found: + + | Column | Meaning | + |---|---| + | Include | Untick a recording to leave it out. Recordings that fail the check cannot be ticked. | + | Recording | The folder name. Point at it to see the full path. | + | Length | How long the walk was. | + | Check | **[OK]**, **[!]** (worth knowing, you can go on) or **[X]** (cannot be processed). Point at it for the details; problems are also listed in words below the table. **already done** means this recording was finished in an earlier run and will be skipped. | + | Output folder name | The name of this recording's results folder. You can change it. Two recordings cannot use the same name (upper and lower case count as the same). | + + Folders that are not recordings are listed under **folders skipped**, with the reason. Below the table, the disk + check tells you how much space the whole batch needs: the temporary space of the largest recording (recordings are + processed one at a time, and each frees its temporary space when it is done) plus the results of all of them. +5. **Process.** Click **Start processing**. You see the progress of the whole batch with the time left, which + recording is being processed, and the status of each recording in the table. Your computer will not go to sleep + while the batch runs (the screen may still turn off). Leave the power adapter connected. + +If a recording fails, it is marked **failed** and the batch goes on with the next one. Point at **failed** in the table +to see why; the recording's own `easy_gui.log` has the details. + +**Stopping.** **Stop after this recording** lets the current recording finish and then stops. **Stop now** stops +immediately and discards the current recording; recordings already finished are kept. + +**Continuing a stopped batch.** Choose the same folder with recordings and the same output folder again. Recordings +that were finished show **already done** and are skipped; the others are processed, including the ones that failed +before. This also works after a power cut or a crash. (Each finished recording's folder contains a small file, +`easy_gui_done.txt`, that marks it as finished. Delete that file to process the recording again.) + +**What you get.** The output folder contains one folder per recording, each with the files described in +[What you get](#what-you-get), plus two files about the whole batch: + +| File | What it is | +|---|---| +| `batch_summary.csv` | One row per recording: recording folder, output folder name, status (`succeeded`, `failed`, `already_done`, `cancelled`, `not_run`, `excluded`, `skipped`), processing time in seconds, number of raw points, number of points in the point cloud, results folder, and the error message if any. Opens in any spreadsheet program. | +| `batch.log` | What happened in the batch, with times. | + +**Without the window.** The batch can also be started from a terminal, for example on a computer without a screen: + +``` +easy_gui --batch +``` + +On a Mac the program is `"/Applications/Easy Forest HDMapping.app/Contents/MacOS/easy_gui"`; on Windows, +`easy_gui.exe` in the folder you extracted. It follows the same rules as the window: every recording that passes the +check is processed, and finished ones are skipped. Press Ctrl+C once to stop after the current recording, twice to stop +now. The program ends with exit code 0 if every recording succeeded or was already done, and 1 otherwise. + ## What you get The output folder contains: @@ -214,6 +275,10 @@ The check (step 3) explains problems in plain words. Here is what to do about ea | "Not enough free disk space โ€ฆ" | Free some space (the message says how much is needed), or choose an output folder on another disk with more space. | | "Free disk space is tight." | You can start, but make sure nothing else fills the disk while Easy Forest HDMapping works. | | "Could not determine the free disk space." | Usually harmless. Make sure the disk has plenty of space (see the rule of thumb above). | +| "The output folder holds an unfinished earlier attempt." (several recordings) | Just information: the recording did not finish last time. Its old files are replaced; its log is continued. | +| "Not enough free disk space for the batch โ€ฆ" (several recordings) | Free some space, choose an output folder on another disk, or untick some recordings and process them later. | +| "The output folder is inside the recording โ€ฆ" (several recordings) | The folder with recordings is itself a recording, and the output folder is inside it. Choose an output folder somewhere else. | +| Output folder name shown in red (several recordings) | The name is empty, contains a character a folder name cannot have (such as `/`, `\`, `:`, `*`, `?` or a vertical bar), or another recording uses it. Point at it to see which. | **Problems during processing** diff --git a/apps/easy_gui/docs/DESIGN.md b/apps/easy_gui/docs/DESIGN.md index 6314ab17..452de17d 100644 --- a/apps/easy_gui/docs/DESIGN.md +++ b/apps/easy_gui/docs/DESIGN.md @@ -8,8 +8,8 @@ the code keeps the name Easy GUI (`apps/easy_gui`, the `easy_gui` binary and tar Version: Easy Forest HDMapping has its own version (`EASY_FOREST_VERSION` in `apps/easy_gui/CMakeLists.txt`, equal to the `easy-forest-v*` tag). Window title, README.txt and easy_gui.log show both, e.g. "Easy Forest HDMapping -0.1.0 (HDMapping 0.105.0)". The LAZ header's 32-character fields cannot hold that: generating software is -"Easy Forest HDMapping 0.1.0", system identifier "HDMapping 0.105.0". +0.2.0 (HDMapping 0.105.0)". The LAZ header's 32-character fields cannot hold that: generating software is +"Easy Forest HDMapping 0.2.0", system identifier "HDMapping 0.105.0". Contents: @@ -22,7 +22,8 @@ Contents: 7. [Validation](#7-validation) 8. [Experimental option: place every point by its raw time](#8-experimental-option-place-every-point-by-its-raw-time) 9. [Resources: time, memory, disk](#9-resources-time-memory-disk) -10. [Status and next steps](#10-status-and-next-steps) +10. [Batch mode](#10-batch-mode) +11. [Status and next steps](#11-status-and-next-steps) --- @@ -41,6 +42,8 @@ Constraints: as before. - Show progress and report errors in plain language. +Since 0.2.0 it also processes several recordings in one go (batch mode, ยง10), each into its own output folder. + Not in scope (v1): step 2 refinement, step 3 (multi-session), georeferencing (GNSS, control points), and rigs with more than one lidar. The pre-flight check refuses recordings with more than one lidar id. @@ -74,6 +77,8 @@ Code in `apps/easy_gui`: | `src/forestry_preset.*` | Writes the step 1 parameter file with step 1's own `TomlIO`. | | `src/export.*`, `src/las14_writer.*` | The export (ยง6) and a streaming LAS 1.4 point format 6 LAZ writer with a hand-written extra-bytes VLR. | | `src/pipeline.*` | Worker thread that runs the stages above and publishes status for the GUI. | +| `src/batch.*` | Batch mode (ยง10): discovery, output names, done markers, batch disk check, and `BatchRunner`, which runs `Pipeline` once per recording. | +| `src/sleep_blocker.*` | Keeps the computer awake while a batch runs (ยง10.5). | | `src/easy_gui_main.cpp` | The window, plus the command-line modes below. | | `tests/` | `easy_gui_tests` (doctest). | | `scripts/check_export.py` | Acceptance checks for an export (ยง7). | @@ -107,10 +112,11 @@ Quirks of headless step 1 (`lidar_odometry_step_1 | Mode | Purpose | |---|---| | `--headless ` | The whole pipeline without a window. Exit code 0 success, 1 failed or refused, 2 usage. | +| `--batch ` | Batch mode without a window (ยง10). Exit code 0 when every selected recording succeeded or was already done, 1 otherwise or refused, 2 usage. Ctrl+C once = stop after current, twice = stop now. | | `--export-only ` | Only the export, on an existing step 1 session (about 2.5 min instead of about 22). | -| `--check-band ` (with either of the above) | Test mode for the alignment check, see ยง7.3. | -| `--place-by-raw-time` (with either of the above) | The experimental export option of ยง8. | -| `--screenshot [recording]` | Renders the window for about 2 s, saves it and exits (for documentation). raylib saves into the working directory. | +| `--check-band ` (with any of the above) | Test mode for the alignment check, see ยง7.3. | +| `--place-by-raw-time` (with any of the above) | The experimental export option of ยง8. | +| `--screenshot [recording \| --batch ]` | Renders the window for about 2 s, saves it and exits (for documentation). raylib saves into the working directory. `EASY_GUI_SCREENSHOT_START=1` starts the batch once it is ready and `EASY_GUI_SCREENSHOT_FRAME=` takes the picture at frame n (30 per second), to capture a running or finished batch. | **Output folder:** @@ -123,6 +129,7 @@ Quirks of headless step 1 (`lidar_odometry_step_1 | `work/easy_gui_step1.toml` | The parameters step 1 ran with. | | `work/lio_session/` | The step 1 session (`session.mjs` etc.). Kept, so it can be opened in step 2 for refinement or georeferencing. | | `work/staging/` | Removed after success unless "Keep intermediate files" is ticked. | +| `easy_gui_done.txt` | Batch mode only: written once the result is complete (ยง10.4). | After a failure or a cancel, `work/` is removed, except that an export failure keeps `work/lio_session/`, since odometry took most of the run. @@ -395,7 +402,116 @@ Measured on the test recording (117 M raw points, 10-minute walk): The pre-flight check therefore estimates 120 B per raw point of temporary space plus 2.25ร— the raw LAZ size for results, always reports the breakdown, warns below 1.5ร— the estimate and refuses below 1ร—. -## 10. Status and next steps +## 10. Batch mode + +"Several recordings" in the window and `--batch ` process many recordings one after another, +for example the plots of a field day overnight. Each recording runs through the unchanged `Pipeline` (pre-flight +re-check, staging, step 1 as a child process, export, clean-up), so its output folder is exactly that of a single run. + +``` +/ + batch.log what happened, with times (flushed per line) + batch_summary.csv one row per recording and per skipped folder, rewritten after every recording + / one per recording: the files of ยง2 (+ easy_gui_done.txt once finished) +``` + +### 10.1 Finding recordings + +`discoverRecordings` looks at the parent's direct subfolders, plus the parent itself (listed first, so a single +recording folder also works). A recording is a folder in which `classifyRecordingFiles` finds at least one point cloud +and one IMU file; the pre-flight check judges the rest. It is deliberately not recursive: a recording is the folder the +scanner wrote, and a deeper search would also find step 1's own `cache/`/`preview/` folders of old manual runs and +Easy GUI's staging links. Other subfolders are listed with a reason ("No IMU files", "Easy Forest HDMapping results, +not a recording", "Recordings are only looked for one folder level down"). Hidden and system folders (`.*`, `$*`, +`System Volume Information`) are left out silently. + +### 10.2 Output names + +The default is the recording's folder name, made unique ignoring ASCII case (the default file systems of macOS and +Windows are case-insensitive) by appending `_2`, `_3`, ... A parent that is a recording and contains a recording of +the same name is the common collision. A name is also passed over when its folder already holds files other than this +recording's finished result or an unfinished attempt. Edited names are validated for both systems (no +`/ \ : * ? " < > |` or control characters, no trailing dot or space, not `.`/`..`, not a Windows device name such as +`CON` or `COM1`) and must be unique among the recordings that will run. + +The default output root is `Documents/Easy Forest HDMapping/`. Unlike single mode it is reused when +it holds an earlier batch (it contains `batch_summary.csv`), so that choosing the same parent folder again continues +the batch. + +The batch refuses an output root inside any recording (`batchOutputProblem`): `batch.log` and the summary would be +written there. The recordings' own output folders are covered by the per-recording check (ยง5.6). + +### 10.3 Checks before the start + +Every recording gets the single-mode pre-flight check against its own output folder, row by row on a background thread +in the window (the recordings may be on a slow USB stick). A recording that fails cannot be selected; one that is +already done (ยง10.4) is shown as such and skipped. The batch disk check needs + + the largest temporary need of any one selected recording + the results of all selected recordings, + +because the recordings run one at a time and each frees its temporary space (ยง9) when it ends. Thresholds are those +of single mode: refused below the estimate, a warning below 1.5ร—. Disk space is checked again before every recording +(the pipeline repeats its pre-flight check), so a disk that fills up during the batch fails the affected recordings +with the usual message instead of crashing step 1. + +### 10.4 Failures, done markers and restarting + +- A recording that fails (check, step 1 or export) is marked failed with the pipeline's plain-language message, and the + batch continues. Its output folder keeps what single mode keeps after a failure: the log, and the step 1 session if + only the export failed. +- A recording that succeeds gets `easy_gui_done.txt`: version, recording path, a fingerprint, finish time, duration, + raw and exported point counts. The fingerprint is the number of file pairs, the total size of the paired files and + the first IMU timestamp. It identifies the recording without its path, because a USB stick can be mounted under + another name or drive letter after a restart; the IMU timestamp separates equally large recordings. +- Restarting the same batch (same parent, same output root) skips recordings whose marker matches. Their summary rows + say `already_done` and take duration and point counts from the marker. +- A recording whose output folder holds only Easy GUI's own run files (`easy_gui.log`, `work/`, `point_cloud.laz`, + `trajectory.csv`, `README.txt`) but no marker is an unfinished attempt (failed, stopped, power cut). The batch runs + the pipeline with `PipelineRequest::resume_unfinished`: the pre-flight check accepts the folder + (`runPreflight(..., resume_unfinished)`), and the pipeline removes everything in it except `easy_gui.log`, which it + continues. Any other file in the folder makes the check fail ("already contains files"), so nothing a user put + there is ever deleted. Single mode keeps refusing non-empty folders. +- The summary is written to `batch_summary.csv.tmp` and renamed, after every recording, so it is never half-written + and is current even if the batch never ends normally. On Windows a summary that is open in Excel cannot be + replaced; the batch logs a warning and continues. + +Summary columns: `recording, output_name, status, duration_s, raw_points, exported_points, output_path, error`. +`raw_points` counts all point records of the paired files (including no-return records), as the pre-flight check +does; `exported_points` is the size of `point_cloud.laz`. Statuses: `succeeded`, `failed`, `already_done`, +`cancelled` (stopped by "stop now"), `not_run` (the batch stopped before it), `excluded` (not selected, or its check +failed; the error column says which) and `skipped` (not a recording). + +### 10.5 Running + +- **Stop after current** lets the running recording finish; the rest become `not_run`. **Stop now** cancels the + running recording exactly like single mode's cancel (child process killed, `work/` removed, log kept). +- **Sleep.** The batch thread holds a `SleepBlocker` for its whole run. On macOS this is an IOKit assertion + `kIOPMAssertionTypePreventUserIdleSystemSleep` (shows up in `pmset -g assertions` as "Easy Forest HDMapping is + processing recordings"; checked, and released when the batch ends); on Windows + `SetThreadExecutionState(ES_CONTINUOUS | ES_SYSTEM_REQUIRED)`, which applies to the calling thread, so it is set and cleared on the batch thread. Both + prevent idle sleep only: the display may turn off, and closing a laptop lid still sleeps. Other platforms: no-op. +- **Progress and ETA.** Each recording is weighted by its raw point count. Within a recording, odometry counts for + 90 % and the export for 10 % (ยง9: ~21 min against ~2.5 min). The ETA is the processing rate of the recordings that + really ran (already-done recordings take no time) applied to the remaining weight. +- `BatchRunner` drives an abstract `RecordingJob`; the real one wraps `Pipeline`, and the tests inject a fake one. + +### 10.6 Tests + +Unit tests (`tests/test_batch.cpp`, fake `RecordingJob`): discovery (sorting, nested folders, non-recordings, hidden +folders, parent that is a recording, with and without a trailing separator), output-name validation and collisions +(case, invalid characters, foreign / unfinished / finished folders), done-marker round trip and path independence, +batch disk arithmetic, output root inside a recording, failure-continues with summary and log contents, restart +skipping finished recordings and retrying an unfinished one, a folder filled after the check, stop after current and +stop now. `tests/test_preflight.cpp` covers `resume_unfinished` and keeps `holdsUnfinishedOutput` in sync with +`OutputLayout`. + +Integration test (`EASY_GUI_TEST_RECORDING=mini4`): a parent with two copies of the recording (symlinked files), a +recording that passes the check but on which step 1 fails (one file pair cut to 200 points), and a non-recording +folder. Both copies succeed with identical point counts (2,566,166), the broken one fails with step 1's message, and a +restart skips the two copies and retries the broken one. 62 s on an Apple silicon laptop; one `mini4` copy takes +~30 s. + +## 11. Status and next steps - **M1 (done):** pre-flight check, staging, step 1 as a child process with progress and cancel. - **M2 (done):** export (ยง6). @@ -405,6 +521,9 @@ always reports the breakdown, warns below 1.5ร— the estimate and refuses below 1 gps_time, reported twice: as step 1 does it (no-return points removed by the load filter *before* the ramp) and with those points kept, to quantify how much their removal contributes. - Near points: counts, range histogram, and sensor-frame density below 1.5 m. +- **M4 (done, 0.2.0, not released yet):** batch mode (ยง10). The macOS side was run in the window and with + `--batch` on `mini4` copies (stop after current, restart, sleep assertion). The Windows code (`SetThreadExecutionState`, + Ctrl+C in `--batch`) has not been compiled yet; the next Windows CI run will. - **v2:** gated pose-graph loop closure (ยง1). - **Release builds:** `.github/workflows/easy-forest-{macos,windows}.yml`, on `easy-forest-v*` tags and manual dispatch; the packaging steps are scripts in `apps/easy_gui/packaging/` so they run locally the same way. diff --git a/apps/easy_gui/src/version.h b/apps/easy_gui/src/version.h index b4ef5bfb..1f05e460 100644 --- a/apps/easy_gui/src/version.h +++ b/apps/easy_gui/src/version.h @@ -10,7 +10,7 @@ namespace easy_gui //! easy-forest-v* release tags). HDMapping's version is a different number, so user-facing text names both. inline constexpr const char* kEasyForestVersion = EASY_FOREST_VERSION; - //! "Easy Forest HDMapping 0.1.0 (HDMapping 0.105.0)": window title, README.txt, easy_gui.log. + //! "Easy Forest HDMapping 0.2.0 (HDMapping 0.105.0)": window title, README.txt, easy_gui.log. inline std::string versionString() { return std::string("Easy Forest HDMapping ") + kEasyForestVersion + " (HDMapping " + HDMAPPING_CONFIGURE_PROJECT_VERSION + ")"; From 4a5282ce818c831dd0d76b021381d5fd96d900fd Mon Sep 17 00:00:00 2001 From: Martin Mokros Date: Mon, 19 Oct 2026 12:33:53 +0100 Subject: [PATCH 23/23] Easy Forest HDMapping 0.2.0: screenshots of batch mode; DESIGN.md: Windows batch code builds in CI Co-Authored-By: Claude Opus 5.5 --- apps/easy_gui/README.md | 4 ++++ apps/easy_gui/docs/DESIGN.md | 4 ++-- apps/easy_gui/docs/images/easy_gui_batch.png | Bin 0 -> 15886 bytes .../docs/images/easy_gui_batch_running.png | Bin 0 -> 15867 bytes apps/easy_gui/docs/images/easy_gui_window.png | Bin 17165 -> 11011 bytes 5 files changed, 6 insertions(+), 2 deletions(-) create mode 100644 apps/easy_gui/docs/images/easy_gui_batch.png create mode 100644 apps/easy_gui/docs/images/easy_gui_batch_running.png diff --git a/apps/easy_gui/README.md b/apps/easy_gui/README.md index 11faeffa..0635d242 100644 --- a/apps/easy_gui/README.md +++ b/apps/easy_gui/README.md @@ -132,6 +132,8 @@ When it is done, click **Open output folder** to see the results, or **Process a After a field day you often have many recordings, for example one per plot. Easy Forest HDMapping can process all of them one after another, unattended, for example overnight. +![Several recordings: three recordings found, all ready to process](docs/images/easy_gui_batch.png) + 1. At the top of the window, choose **Several recordings**. 2. **Folder with recordings.** Click **Choose folderโ€ฆ** and select the folder that *contains* the recording folders, for example a copy of the scanner's USB stick. You can also drag it onto the window. Easy Forest HDMapping looks in @@ -155,6 +157,8 @@ them one after another, unattended, for example overnight. recording is being processed, and the status of each recording in the table. Your computer will not go to sleep while the batch runs (the screen may still turn off). Leave the power adapter connected. + ![A batch in progress: two recordings done, the third running](docs/images/easy_gui_batch_running.png) + If a recording fails, it is marked **failed** and the batch goes on with the next one. Point at **failed** in the table to see why; the recording's own `easy_gui.log` has the details. diff --git a/apps/easy_gui/docs/DESIGN.md b/apps/easy_gui/docs/DESIGN.md index 452de17d..17bc1ac1 100644 --- a/apps/easy_gui/docs/DESIGN.md +++ b/apps/easy_gui/docs/DESIGN.md @@ -521,9 +521,9 @@ restart skips the two copies and retries the broken one. 62 s on an Apple silico gps_time, reported twice: as step 1 does it (no-return points removed by the load filter *before* the ramp) and with those points kept, to quantify how much their removal contributes. - Near points: counts, range histogram, and sensor-frame density below 1.5 m. -- **M4 (done, 0.2.0, not released yet):** batch mode (ยง10). The macOS side was run in the window and with +- **M4 (done, released as 0.2.0):** batch mode (ยง10). The macOS side was run in the window and with `--batch` on `mini4` copies (stop after current, restart, sleep assertion). The Windows code (`SetThreadExecutionState`, - Ctrl+C in `--batch`) has not been compiled yet; the next Windows CI run will. + Ctrl+C in `--batch`) compiles and its unit tests pass in CI (run 39163046329); not yet run on a real Windows PC. - **v2:** gated pose-graph loop closure (ยง1). - **Release builds:** `.github/workflows/easy-forest-{macos,windows}.yml`, on `easy-forest-v*` tags and manual dispatch; the packaging steps are scripts in `apps/easy_gui/packaging/` so they run locally the same way. diff --git a/apps/easy_gui/docs/images/easy_gui_batch.png b/apps/easy_gui/docs/images/easy_gui_batch.png new file mode 100644 index 0000000000000000000000000000000000000000..567670445cf5e1502024284458ed124633b51d35 GIT binary patch literal 15886 zcma)jc_5T++xJ*g*;?+9y>6ixqR6heTdZXXF_wg+F)_w6)>7n76f!iHR7RGJbtW^$ zmXdX{FEdD%VaAYUOvd(IdhYvup7(j5_xrv-Fy=b1^SZ9{I+ow@JAP-vRjW%v`;P7d zfj~l6E}PnbKs*mYAg*Nre&CzRB2X6yWYB-b^t@f@2y+x#;_nbC{&R;^hdccZ<=Ifj z@2p+a;(MpS;KWtrDZK9Q7w&ahq&>56>D||N=P--s>Ajxd!O zJ5#eKXCGXL-ZU0?aI9IC=TR>&=KXY`^kyJaSL$x;xO&}+ygoizHiyI>@xzjGx0W}q zTw>=`=5AHEu8nN$lvEEI2H(3k69L>_**V2y$Am@5ZF@nW7{1|gBk2||9uTNOW3K=g z2*e9K2S{M|lc*>+2qY)R3j!G%3jjl7M1dn*@+(0>LBAgYf!yP)rucco!^7X~wF*Y4 z6i@mRcc6@tZCqwuKka(7n;VQW&M9l(3k&ni12}1R`No~Y2b#$8GefNI2o(yv-^pcKjP&2RziWt zo~KJRv0l9eUoJvBzD@pi2aYbhZ##;#Ono!8pveyFq{iqWda@3-80CMH1%cA%hZ-9* zSK{M+Jnx#wNSuJhe+lsdE;w0PAC3GV{{~pS=FKq^aUS8Pzn@@BVJd)?;qpjBM)FSk zT(!pm3+^oeeD&WJ8#sr%DP@7c0Ovo?P_rc$6Bh`@T0+WmlD0A(H48}!(3+Oht4mlq zZtbos0uG+m!K~DfJ%>MqVI%H4?RiK!>aVR&WHpLJ-5Y2zVy$*gUx*%rggnKl1SWfa znRJ^Tq_;fvz)#*a`}izub9dO0!P+c+#_>^e|H+M#OA0If1~R{_L10ln@$t?!b9Sb_ zd>Z|h1$(De?=iBuc9KPz=!9x*jjd5CI?wb=`y_Zl*BoA`Su(01GMVEt>Dw5*KM_JV z?^?p7L)(z(XiBiYB?5nkh@cd27!-t*mCCRpWu~R9(Y5n|IVA&r=rxfLFofmiapa&rJ+pOWP=^6M!SegwDjxE+1L?0a7rs}IAc2X+IVCZ=(O7Aqa!wrrp*=RRdwhSYjC|mi$khWGvsh6v+dWo5vSzT2pgGxwq4q>$E z94zc2JbjQteFG1<1C_-9~cYc)bKyb9ea!Kcp@*Y;cza)f?7_I1Ge*3Cj02v|o z(5{(fllQvB*T#j^e11_EJ*QDs=`XA7RmEJ&p>k}*eowC(<36-c0PP=o^Ly%LyR)@W zTxR$+S-;pC)lqw~#5$y)dD_O|WdfEUsTXmvaB)QmPt(4*>EduF2XQ5Qj?)vQVjFs- zFqv>Be+}MBCM6d}rQbG!_c|<_L&$$Nafb_R%tjui{C?0J+GkUrIf{^B+3y)W^VBF3 zzH>r1pM6)*7ps?Em}Wk=nPs*iu|K!OZ{J{&jlnC>wVO7Xr`NWd8^$*gk&vb=@LNXZ z$(x)Xkeo~KOg;@o^5?WQjA!NzSl*g#IA)aZf`~`77#Ty8D3ZC+1;N^J=t&BQ;jh}q zBcX5=ZC{fqr*TA?FDZ)|+=}Xx|E|!3Li^MlYFO>IS}m_9uxu2KC0^I{Nt>gP5JDEc z2c=tXCi{_YX8EAUfdsyW;@no`)QQXyq0--}^MO(Js4z> zLTzn*@=5;wNKrOvMJ>{Lp(`d3i1%C*d@X8(bKsg~p@7!uK`11u`x7?N92;s<`(yRZ z(k5Q!&lNS25=!y14i6|s{>{?z*y}?}McSK=UeC%FQrfXk+F?Ie0ybZpPX$`Mceo0S zGM<%>sBy@tRY;QdV-HPu4Q^-t>^dGFh`cb+zCN^cXz9;Xql4JGCAF4ftF85OQQuQS zIosrx%H++KpKDvW4nX=5-4D$1$@`Bt~xh4|YOMzvndqzqHmQu~iCD7p} zXQ=uInZtG8r;^X4uOKB(`GG(gUo{_!L6Bi=Y0@%1bfWOmDV|4;j2g!+sHXeSe7~bx z&d6kW4O5HK;WXeKZU2$;a=@3UpP!``zLjeZrZ|lML`@D~+AhOoy~#rrDeusu7GEwZ zAluVjv4IRJ5a^JA00YTBw(iu_{+jU>$8vwAeD!io7_73B>IG%vGuMhFH?&%MF!A@M zmnC85u8Z~sV^TXzHlw{pIkZ)|of-k#r%U92WM1r$zbpcZDeuxiZs$Mt6#JtkM50+V z)vHQKgR)vOwHDLIgxL+Wt>=s7CQl|9d~C6Hd1=?hX#B%g#MVAJ0KI(jS{L1ZB*2bu z0|e@jliM5%$$feVS?Ham*skq- z4)jlTvp11K(?8j@-Rfes-8v5fovxQ@63gAXE*0MSN5FReq8D>&D8c+rcv4_o7IZ8$ zefGsWiuXcA9xR7=7f#B`)nn&=pWBGS24K1i7B@)y)!t!G(Fj)_m`#s^S0YnCm!%pCLZi_g^M(bJp2(0~Y9quOUL><&C& z{VD#FxtgKuc-Rl~CFVx7Q4Vl_X_`PPrus;1)%nP7L{Ih))+?{C(8%?N$ypmOU6Mu( z-lX?FsdVZZuiIGxPBM}OUa(O~R{yDw7q~#_M@8#aNn+pUV4Fq(bM}KR3Me;+jbX8z zs57pi%M<5zdY%O81Zz%uf~ONDLb(7yRla49Q2PuYkC4mR8PNWoa%x;u5lo?EWK1am zXLJK+giJ3yRQA~oMFYHo<4W84pDLbIO(t9C*QP(S0+thD?6`a)>bJi4J8wrQ6tu;Cduc`w&uiru@*q&+rvx3| z*arX#j7i{qcx+W4-}icl^%4NU^*H9oR4Zc=0H)Ss-#n8I4%4~}4ijfv{q||Se1OHsK_fRTDOAC! z5&k@i;rL-w2j+1_Cc^i2>OddhbEubU&?E)LWJ1;{5a=r&^K;%bG^qsTfn~I4^pE8e z&b(b)z+KsJTrPnwfM%MRj&i0Tw{F zSgKde^i>UzcAg*zWDft^fCC@=p?T&oC?+33(j^n13` zzUZy1MVsYECNNQ)9&2aUJI*?Y(5}J!Lpqs*GmplASLLJz8mCYsEip8V$1HT=yJIai zl8V5S&m@tey7Htx_gS4wo^f_dcOidv31&o7^u=CokA*;vnncc>V!}W8-)VC9ap7qDZgltz1g#SNbJ8pg#PD?nBH6779&o4HwW z;A^O5)f=eQe(6hH%ULiUbQ@`boyAFU4ncQev$m!O68g6ClMc_mp%(PTrETMMwV99+ zLJF;WW?U<|z&z@O65EC`T}8B9AD+H(2IJ?_imNquEx?&W$#06zh;+m4>4a3YHe}H+ zPGD>%YE&gat-5_9$A)Q6kAy7T>n(>ofp-mOvqeiaUKe$LsgWD_EuziRobn2%?8#Ug zOsmV~f>ra##&>qa-1#-Wx*bVWa@Nk9gqd*WRi)q~1%ZYgd6urrH9c(ByX1(uVtiR2 zF5bVAvv~|b!O@~dr8xRWZKE*!)gx0sWSSIQeiY=?Q8No(4iq0P>JEBsp~Q&ADEW($Lfsn#Jl;W%T17t7u}-LY5*#)dcHIG0H5le= zbUJ-smZsF3KCO*&1DHQ=+(aYDE7QU)E^hebigDanTA#)1{ZG&ZwEMae|T7#K6DZn@e?eBfLRW5MXebTg8BIAx=(C`nO*pSF!uN(6dxB2bn#Pdq0Nm|HM|v^3;`{VZz0lV2 zs(OekFyHEEWf-<&=frK33JN>TR9DibArxyMH)++Ho?-LXgJUl5)p`k@fnXjjro7LPu`{W)))ptZESb>WO8WQLD-+}KvBNoG&>hpZfbeJuaN`wTSAo(h)Qat5MG#yRgiN7}WBcU9&X+ z$*qcF+$<>ix&3D>;O}7b=J|Qe0YGKbzep4|1(*u<5NF6+w$Wwup;%0o80fAJf_}2-G zsj@Ha_^Iqx5y($pVi$G!n1<_Z!}v)m0!d#RUmFVS2@DEKt8(=>w9K#jb(NLadi2=R zcB{g6opqEF;{y!Yvf(2+KTVy7)qRVgjDHrFvtoXlr*Ic{o|$^<3xipIR1WDf2>rf-q^V9ZQ!*v}mlVcVjlQMo=Ok&=YBmqi0P{ z;`x#hq(7nQFJ=o}_}nRmXksUnk?*g0mSjtpF4>6a?cCuk+V8+9mDk18`=!kjChy)z ze;I()2dX;L zWEyb{nv{rLQ= z<%f5E?%6-6U<82efZE`j6cw}ytLJRz51sh2rY{lTfR3>4GpSxy?q9uwJ5JZ%IJWwg z3nY4oAx{J z1i+xjj}H;|@elH3bdH`Zh0 z)L#$5qk5d3sykV=y*vPQxnD@YI;YmKl(fc~5;9yHG6`6qnhf8kmfA_1pmI8Jb0bG; zgPHcDI9IwB9O*(C5Cef;8Cfn3Ck!o3DI;@k(hd#1<*nLfL7SdkYy4KLw2RzCuj_`C zj*5Og*ywIm_Cbeg(@=w!BPDYH5ZqA~UlwF--~YQ3BN}!CMuErLn<&BY!)KvT0R3j= zCm#Utbh_uBJw9EsGC3~}#Z?0*?XLjHzYsMHup;jGU)Y@cLX%+D{|6HM#k+p!@9yrN z*MK{}pNN)R`*PCJ+v{+w-e#axnN5Z5g_Fy{i2Qaf+pfwURwZ4ksI54U;os_KjKUQ(YY!=JP&O$z zPiXJOg^I*LAk0pyPi=D?u~S;;t~r}V_rlt)nl*OVxaMbMdPRLH^s+wzz0mkFsk%=K z6(QR*ya7>X||}Q9T{1bt3YY&gA*1RzOrm{1-d^9?)3$kowAk!y_`#G z(xG|JJ%0%QATnozMUONZmU59qBrFZW%%y|+KfS?v7E;_2ZnshKx!p;k}VKbe)rZ|cH;W&ga6k;{3l@F1u=-7BxleUpy__;LvYdGFlZT;!_ zHoS*PBt>U+8^f7@yyq8f6B20^X95|%y`w#WI42I7ZXVljwy8+<)@ z-KLsMU42>@HNb>-Anuif8DN%A*&7w-XlK)(IusJltX98C_Uk&_{@PsUEe@e@B7QC~ zD{@>+Fu)20rcnKMz})Tf41_G(AhNGAjrNUV)!gfaHYbQEm)~_rl9tNSi4G0Dep~;! z{QWQ1_hj^v8`qykpz0Oo7jtETf#U3gFIP;)V7R~>%%xINc75+dxVTHfdD0P4!o>L~ z?ij|IHshSnc0Lkkd7f2?WEu z3}enrrJ0OsZQMZxE>|}D$rW{BsRX@S(!3yphnitCZx%kkDaGGfAb+$Uy?s^{{Sh9n zur{dW_Bm=hks0x>IJR)ZFEuHTvH5lS>M*0)E_zAIM;Q6MDUn( zEmCH$S#4#F{Xy)j#^2k55$`xqgf01FzM5nNKgWg(7eZkd76>|In-@7C@4Jnj_D9WW zaI+D{8Va73&z-78k0VSGHyE%?s{N%O|& z{GD)}bk4&}PIJ+$3L~Cf0$uIe-u7LT2O{?_)BIRkxVwLa4@j;FI^>ys=iJOX1cX4% z3;*o$7pQp~e%E)!Qz{ zdZaV(2uJDCNe4)tx2(Y4ITx4|LmlI? zkUf9ULf$x|J;0wbJ|o0F7(0cD7_P2Cwc60X4KXBuL7!a2!UG zS@9inn2@csUocz<)tal{5c9nqkW-E`prRTV}?3D48aLYZ?ZP zDMEK(pq~@tH=p`yB!j1~veyYQfSHj;uiI})nH;rgM^aFH@R!uzwUb-Yg2|`21q7b3 zb+v>XyfOkG^BL$l4(j!=ydwk;jaW=OZx|+(2kE?BOHVW)Gm@bDe7`W6FWW1dJe5(( zFL;bYnWk)slkxO?zn9XsFptF7cy*0jb18V3R%aIqUa6Tcu@eY$*|_YMOM2w_&fGS% zvkBbe{{1(DAW(BCQb3!hGA-u~0YOjiw;V`;O982{lK^Cgcgw!IT|F)5(%nyZAvZ92 zxln%#cy75bGOCd7NFVUqw`_yYB&uk~+&RQ+_ntECR8WLKFY8u=SqFkJGfuyns6omF zSyzx|5n;XTx91>T^+2T##4a$8yJL&6a95U*zP|JN{lU6vQlN4$JLvt6eBUhdRjIOU^eir<-FO#;&JgK;I<&+` zO~XFb*KEjsMhv>BUIIY!&DV?a9o5k*PnRAg@AUqBpB#BIij}(+tiLk9^1PQ)O|QPO z6Agh4;>h;z9q^>iv5F$rhzg^p7nKqRZXs~OdzbJ|V^si8s@@xOZ`#0UW$kB8NZr|} zm4)!{3(+jSYj+%PncWi?`glwmZ(Cgg{&JV7q8JI~9>X*g-x(GeU0Z0IY`@|LISL@y zN4yW~S>@KBV4Ex9-wR>Uig6jUw|*Nc*>OgGPcy_h{g!o>1xm8F-5o2j=V82Cz0r2p z9z!X%!H;{-T^p3XGWzm;0%ltsi;xqT!|wh0zym&Bw>-HrI~gq%&D@fGoXc6S&y^NT zr)Seo3CwAv`k-=fg>@%iD+>XAa*O)-AKHs`ircOAKY!llNwi@=4MQ>rCCv{t5F3oZ zlKl3@-ByFT!Qa*tJ|VK(=<@xfy)|qk?Y14eqq4_yc+vRqPcBeHkp4Q$+U4gCU`3nq zZyuZn?inbM%HO#*s;eB{txi;uY|wP&hj9p5Y!nE88lR#`Q)Q`lb786;^K$N#eyI!!{2?%xFFbJ^G^nEY@TSOXwMCUmp+#d+lIUglmH?|tB% zUIF}dmoo=u{8?9i{khII_F|G z!i5>Sm>fnal{E@g>p5XbW(CWVxom5_BfxcG1B+*{n0KRXY28UJ^LF%9~3J?Vh57Rtb^`cTpo0M zAX@ApYr0VE3FgVoQSJLW8|_#8&_UN#o%8*wYyC+@p1wB0EmrZogp0{W-J7qo2Ym)clW(6On$R>H>p6t+8{ufUNF7xQBz z_#Yyk2Fs2I5AelDMiFn|nN%-CE_C06LCGcXG*?;k7!J(5)sM(0kC6;ZjfnFJ^T$O1 z1n56yyRc=73QwcT3}zu+L&i^6=iG7soogcFGZmHf7Gs$V@z*Ur>JPHWveC%)+{@qJ zI3^H;N@whG!(W zKo?*hpU*1NGAJ(S>zaNu^FD9`gQqJ(Uq*X@?l*Pq2Lc_y)DbzMZ6J#xl$gRMIB9+M zlrG4Cf*kYs7rqRRC5xkJ6In&d$7(|GknfySy(gjl4}XU&tDjXhf$VJ5M$hTRfPb6c zzv|)oH~Pmd^X%D!Z~2GSG#9%f!XjvT*2L$_Pr}wqQ=GhkRcQLtuFN2{SDwSo*L-Kt#BPS|+gXzC!UGb8QMm>*E%~u(v7-<#+jxy7-bJDT=~f58;nR zpTCZlX=EYr?+2^&6&(*B=H_m)aHwDJ`oOVVOoBzt;V02<+p{GP8lhr8`e4zvb!Qhe zs8xM9&`pfNBQ$gp=SspEAQ8FaUa$$raDE1IIxu{NAgrxgts3(~(4+a2gR7Xs&QSKa zh~Um0mn}@a2GLzKCSyu0EdP>a)n5L=@M9sT13fA5hoGzaa+?dO$)8OCTJmC+QM9}g zm9pd8`0cV{1($n`=Cg&28TMyCaEYzbk5bwi}Si4CPyx6h*Jlg z9W}l=i>wvJ#qZn4^;rJI3n#uugFCA*Qz<tYpl@}5rBhK&rhM&0oNd6T-&Ug27t)>m z?T>@1hI_sv_sYq+u%}WlEdf83U5_RGf7&iRg>6Rvm%n*WRAWh{xjq7FI_sXFgqI(| zcg|noYIyjfRt@AnG}NZKdrc1;fHeKby;ASF63IlJb^4tb|T@glKPioLawe95%gGI|2C@2H&j zn8}BI{Kt=#>a3g!=ss1N^{-xq_5zuAUvmD?j$^MFtt7NHFoCJUoMP}2KkWJL69su4 z0(lzkU&SLe?RP7cxtuE)Vpdou!)++Gw?OUq?rC1`Agqc2V zY^~6cWmo+O&4i*Swh_6$H|pEZ{(e?G@n@=QJ@o!KIJU8K45xK|fJ2}R_}!4@ZGaG`<*u{s7YWrJWy9D3k=afqk@SrnIe0((xvVPS_HeNen^DQW-C&VBbEw!K39 z1yMY_uT&Fnx(S!E!CI=$TXj!p1iZne%Xs$##>O8$J+aPy01c)ZZ(n!131fL`cC!iL+C!>+6uwg5j%kh=q; zTC#t}hKJO1e!{!V;jF&hRdFOg*UgwKUf0i0wC*hJ{CO^Texvc*jt=^lw?R$fM^wne z-fpR4AQdl-G8_~(T2V)tV!2}%oEnSv`o!fpo;W*8m{Y4Q4hB|Phcd6hB@BQYO{LA; zC>Ai2Y_rYSS;y*HCK53&g9X|cYB7#Ko^hb5?NwVT*wwSI?5<{P;Kdt>{Ke^b2zFuX zeldAwVLuQaHIyP&C+o$>BbPmN82tx8rZszSn(-mvQWs9X;}P$EGRGGS5+wn6I{*!8 z4pe-xmlO=h$f|k%(Lkz{(cm=I?8wnTARxKq#RE7fCc{@`;*{%BTj(gms>@K*|HFiJ z;Q`RdUEcr_uc&Bg_~sIP>E|Tvue24x^mF?!FI3;ZXU`yOMR4LjXRfeaoF!**5&(gJ z;bD*Simt4qI>G>~I}~*~GU*g7%%^riZL2f%=lC`$bnUqj;3zWE(tnsXkB#Mw@ETv+ zO*Q*>>VdqI7>w4D*`zOCG;J<)9xr^wb#unxVX?dVvF}1g>-Rs*Rup@u-mkg;?c}o# zCm5^OTcW{cb@WInEF{t`lDPupK3j5eje&~ z=+^sx`esbbkB!o(mFxf(dQ5tJvj^6qUBB7w#1~5ee=8i@c0q? zus`k*<)YmbZcf>4fr)(k88Blwa?A!fPY~38e`8SdDS+cY{J(1R7~lT2r8M$lfBi=vrnnl)wFr z)OlkcP^53WUk%-|-nrOZ?pe8?AATZwxb`P3JgSTt%ou1LmND+Q$NT9S)MIp;^YW^< zbD0CX#J&HarpX@XLJ}OnGd@hDWjn=R( zJXZd#uQNcPFhUCx&JWni`9!7rLl9G~gDfs(JKS=t2bnovtCF5{<{1e zo<42ifi@o-Px(UeN?(_vG_tD7Hg<@&6q?!ewZT`fn-Q9LDq~r}OWcKCo>*soLQS@fx+ZLjAB*(#GodGW&EkAoneP4^o187>Ot%_@G9D^pD{Y}1 zYq(sSEV-FCLXc-TxgqM3H^_OEMg{9Si7J6W1v|NsXVOSX4UWT?=VuoKP7#A0h-0 zEMggJ!wz**TC+NcW7blxGVTgI-mxm$;9Xn*(K|uwe{q10Gnx-iKe*xuL_$Z$f87T4-{K>8Q-ZFpF5yRSaCRD*sR8jcsLR?~ z&`>M58XuDB*2WKtvErIw;>Tt1V@QV8gT!Aw_rj5g;0{bmlD*De_s>qaUE%wM-5o)7 zdt^f83^#_!pW}@-CqB@NimBfKBxUg>;h<#AN|)Ybm)>Hc+eWq96Sp0f8>~OPPfN)q zGK#h6o!rJ*8(GEmNnh&M((e806>ZP(U#rM}8%_xrU>1YPfw5~vV~yaJ6x%vWue6tI zfV>vis)N(=qEyd2wCJM7;J{uj0Qzk%tL+3*!)Vz{=Jl)QOT*+SEspq6hY%j3YUrIq zXYLXc{6}Uui!~d5D>Vc;!ooS*GLQ8e#uX9))vllM z5GG#zgyDJ%PLkD!UIg?U@~|12VGzDF(WWbQnW1Yoeighkqhbi z-PUyYTU#MP%kR7(Mnrt<16@%Yzy!r1j37gKz)%QIhp|j_v<+BFwb$bJf_^ zyF_5ymmgp*Y@qQFrwl$8$_tVcl;BInC9;QGsKZwQm5-=KZ$taj`d5-5(Cg%XYP$Z} zVfgpN&HJk?e(h=z9Te2JK0N}cc_+L^0eb`X5oz<(fCpk54xT@;3$4p&IgKgjJ=&tQ zcs{YV*yVSf%Zjvtt$E^+#-YnotN>>9pd#!?d5g@SaGK1~TN$?(%KD0S*wrPb1WGz! zQpUCs!8eP@f?7gGqtxx)7wS*vbJ=mRhus1yOn_u+de+_PUS#_0Tu_WXzMe4Fl$K)( ze4o7skW=RiaWO}FwI1#D2#kABisYq$C0_#@D1hxSuc20VDs`&P_+;+&+Wo@AJK4kP zS(-*f(B_7_yT!KII7RAIme61_yz~b3-2x%pz-~K7{d3O9 zh9jDEONQPDAQYYew&)2NfbB^Zi0FEteOg}@;VjyNqn*qbtGpA9*f?2U0(L6`lx?nU z%P9svCNv-PmVN(|l=aM1fG}5YjVV*UXTB8;oF~Ksy1JM&T7k+;8P*#eHZ5m)G~@6cXfg6&l98)-f`+ z5I|}B7TRMJ6gjud2S}DW0>9iwH{|(Y?JXiIQqsoAHEUX(23KE(uLCCMs^`(R9US(& z$9c$m*riQNk&HPku4@E@#C&}d$FmVI$}KsFe|j-)xaHJg`q~TP-PyfY0#|gkbK|?n zd9f7)=*z1>%&L?6%j)k=?{VEoXSXRVg44v7HIE(3`T70QVPQ~%#jbXO_H0)(*7R1R z`S%=HpPmvRf)f=5xAnkA_*9n<0nXYE2;X@=k__`~U{B`iJ4U+1ggs}A&7k~jOr{yBZ#4(xrAA{p(*9Akrt?X1R zFWR?rMrcxknK;9?+g-aGTtEG%=G!O#1-Sn*k=nfT6}srpVjrBF_oH~K-x}cz4FcUR z|MT(y(r&VIfGp8}e#LIS|EV3a^YO#N%VNFpL~Il_4VbsN?0@gc{eir5on*}5`i{%V zE1E_y-ltbHeMbbZ+YVv@-6Cb|PIi%}J1x}{2yRhb)Cv?Uu*P`{=d^hyp*xFyPUxz^X&)c4!fA`OYn-_HQaqSH^g1{s*OGKgd5-c2P`t$K z<^69Rx*xYsh|U#jE9Gmep3q-6hj3<|1p{u`25cqzZ|ym^GvarT63(T+x^vP*n31HJ zkNT(x zQi4)LARrcy8mSQi2}%hNL&Ojw354HCFwr~QcR%;N_j#Y+`3FAP`|Q2;+G~ARJvUEU zng06A)?WYs_!Vk)%oYGv;s9V-&OcUyzevAHI}QMlm(XK}Plxp}24J^cI0*UA*PFAC z-I5;J0P8-z-KMeWh~h5^pKjQ{+t|0mY(vwolP@g$ahE!pAML(8s=O<1<*r|{DyK|r zmlG^}KQ2EUWb^3IhRSP&_xEh^IhWjVa06q{`!mIRo#S`KDqeF6oPAD+OP0=+ZEJbJ zg48N(KU8Jl*`~j+pnt&gYnmqomY?5Ho_H=ILJg!2eD)QBPyT}O=C#Ge#R+w1Pc{Wf zTz_@(;>B}%vaa(3?U}JE(0gkki<4m)FIEWuJG2ozqTt#7i_79akG_vQ@_qGT->VPL zN*jI>j@b+@6aJ0T-0S&Mq7mu9=Cw|Q2p@7ARx?sKuzt1r;vzSeJM6JI9%J$O1Q+F$ zpml*9viRn)%WD=V?5Hc-bu%{uQa|D3t93~SAKuPxJ*dvY=H`vC&;Bf!6J_SVxa<1W zSQ`oBxx^&jVJ_Dx&*~r?vgK51!k{P`d0UfprtR~&)xvWWbW&!c#)u#YSonuB{{firhFR>{Q5bf{YrEjxptxF z6BV6*2>@Ic&br>#0HWv$-_Ir5Zz)NVd$(QQhT_i_!C0&YWB&O}a(fro%fqPE0C-nH z-$rJzfbIy_!iga7-t=fWA^`k&D-NM4t=^ufdLO_3{d(PB_fw}<0cIWViII_%&z3Th zA+R&)#5VwL^1(cn$&HKv#ODa!S*ML$b-yVP`9Mpv767tLpj%0~-M5PZRshoMZNIv# zI;<#?As;LM>`_;1c*LrO>d@QL^HwbE%;sJA0l1p(LAR2bt-HL9XfW&U=AVIX*n-x) zkz3Dj!`+JD+OcKP;}Tv?d1VtVF$rs@37qI8IA048F~S|2&E4Ir0nrG zn9@$ltP@+$0S+N|VTu#$NmJ1z=+Npxcn2F7uv4toGfKGQwFgS`nkLVW;3k+EX{9yz zWN!;UZ2PPlcx4(6oPc-Ja{w^HKC+ciGuFp8BNrV=rA(H)B=wykT!J=Fm$x|*0O(?A&73i!|e3<(8Q2bwU{kiMsMuBWG`dqH9C zN=HY>#O2Cyx zjMyd&=hO}8hfT|{dK?zsU#xg%^@q1sW6~|DKjBJm>KUX`P;5~@MR6%Kat#L7TN`kT zQm2&<_R#1s*;HDBfd5B@WA)f zd3=d-Rk#so=#|6OKAaEmX0baxwhA`@jpoygvt<#p2_Nn8RMd`5ob$W#E9}Tf^V^gS zYKjqmGc@CTfCkh2#~$ z-SoW*oJb>|qWnCR%ccmv38Qrrq{73tlyv&Sq)+#0+kC;Sjm)Qa6HxnG@h()9v1L`Q zJJYj!wRZZ>j@acSs{mDK2onLMl3$%C^ zB<71mg3B#u(6Iarohes-kZ11O-F<2=gx(0AAUYh@*E4JjYb@02I<;5QWu%wDIiyn? zda!W@9__-L4Bu77q2%EfrTdUm{2=~wPX(60P;*p5u{VjXR9UX!yT>=)XlB>XfLTML zGt9Cydo&G*8!KD^bcMpq_gKD88d%dpyA@pYBI$&~Fg~1L@KJh^bx^01gXYRImG}HA zVF7D^s9^mKNogt-n(lts@)+`1D20fk4QJ7Q2BP-*7v-0`saW@6ixZuV9I|*Tu*3Bx z>TkZeryb?X90G@SIBcxwxN7j%sQ~8Fc7rHS7`3|Dfq)s3H|C-s1Ymm%r>B6<&CR2i zW{{K;kv{M;mGLpT1lEk^nJxo$V~%re{6VBO!_WAIg)WH$`y~K{V_m>Fuq#x^nFzvyDE;g;}q?3PWb)=updk_tFb(plH7X;v#0R1 zfb#P4FLmghZjO$-8PulhLg2>E>hdy86|6g6E_c8t4m1&&`!^wbK0 zjhpAh$I9LkI}}G(W4B?)UJMkfXJe zPSZ*4szu&GN*~HgSNIxDpY^R7YA9h&W+G@^>GwyZQ6otX*xA|7A6~6o0YtrwoTyc( znq*o>a@UlXyy3IZb8PQOzQn?K&^iy`krW9Njhy!|yp2-C**XSz69*^vD(Kz)Id-l( zQ&zYU2$No@0B>GXwqj$N1T~3l#MfZ8u@+jFT8J#R3~U`>_9t|@yv9vq0t8Ki&8CTV zLusCU21*SO-)UOFO{~fD#P?e*1G7KAJ_Cb358!Eg&6^~H8s98=tkacQ&Q6ud8NlLe zpZ-om$Y?i~bmyhDN`@NPiSy503eVhr2BXpCBmv98+X%6s^<{baNjuFX60_X;%7bEaszq!rp=K?I>pW zK3(*59bLInWEjqap%U*NIcP9Gy=pYnzyv$orwD>pfQlY4IDgPxX);=w+MwnDw|+id zpz`xE5FXAvqT51qv^oSfVG}l-dC4VgejBvB$a%OA8n_#T9>Ei*qliXJg7R zZi)X{Kfw-o_H2n=z!pI=m6e|Ca9jZt?jNTjU>tg(yL>(GYXf=@G!TrMuim5f_2Ib- zSL(K_s;X{TyE6L|*#(y-1pS_O`6F z+94oXtG#3yXr#I8>PtTv%;(NyBl->t?E}&L|HMAvrzb07K)3SH^{=4YE{?SUkoA3Uv*iTCuY)z+ksw&}uNYsH@)WCa@;& z(E0h-MHUMfKY5vcd`8~IYXo?R_I~w$W*$N`JehKw2e3ssk$19^h<2{%pFKGfIejU7 zEUh7Mw33)QBE4^QrhxJjB15u7_?aH#m6RB{tC(PG&nbi?>NtXvFm&On!zY4h^2yS*)!aL!&oN3gs z-KkUq%o-}X_hMoW^G4g<&=bT(Lor*fLM$F^oC<*EdnG!VNV%T-4H+F{;S$3`wT}S6 zTbG-8j>)NPPcRF%+RZ}GslP9LzVv}0-3kC;WRbV)jE`GQ7A|R$lV>=6i7Fu%|O zh?zOJr%I+1muB6GEjsU4=*7v;W2>2K?CW5H)_}auec?kxaVh7hP5)x9ayyvF8PLKR ztWuc2sGg42g?^!g)~2b_h{pp|F|~=6pX4{Xv#X!*+CF3qog~Q(0EZAavjdo<&KXJp zkNdRXH7|qnOLvfJ+05-d`KERQ(5D$g`%<(LKQD{YqBK9vqnMi{{?1a@-R-E&oa4482Dqzy1fp6V zo=#|isw9R!-E~RVhq=GOlAAc#_rHiMS|Km%4Dj0G}fKUGLlbe!=D3 zo;N6>^FtE-FZ7iD?}G;j7(D1Ebz9_P8JQTx=fbrA?%%+WT=;;AA<)&2#!gcvoMQI@ zfZ0t#`SEdr$p#C-0Qi93;RZp}5H)58z7pTHLzG7orGv%54~+XuU|D{%zLwx@^(lQY zCn633M3cqoXg$@IbWdcM?U(qW@+-X^|D;$Y;sAOSfJcnQU@ar{5H0%~K%7c;wsd zg520Xjd~fudA?`eUac19(QZKk!8{r?L2&M~KhBJEMVtIKKe$5vhYVZP*IP&aMxp4O z-}x;n`uukeaWc*Q=*c25MK(Z{Sb4>qbb!fX8oKq8dy{50BN_cv6Nvg#8g86EHBv3` zJVF2e%|Hm*=eUB9ny6sqI(C}@bJzjRc3_QYp!IXD)9++KD?@q?sP8{cP*jpuy|m-E zp=!p`XgPgi`r=ftX?idG&l}G-#VgJYHO>*F}18TQ+Fb&x7n@?UDD=kNAu~V5nP2eq+^%z4S5_U zG1qv0-0=?^nqvK?3aGrKtA!gxm11AOLAqs)BMk+^jfpC({vhH5OtA6cjv2;7o%n9fUx~xlS@$N1=@b!% zTyw23c&GQmL-Xel2Npe{Lr#u^+)QI`=0ih-rnT{WaN9%uKaY11DK0*i8*k4PKPV>= zGZd8+k36inlN?;Gn~V^U0IGHO`@X&==3EP zZR4~XOyy5v4tPcQ^tKh#2xB4kn7g|Urj@8W1oLmnYrvpST2L+(^VLr-P?}w_lcbXJ z)}!4;o(Zmk<$`+?eiQr!fJYW$yP%!UFEPLk!kHz2OMP)=YfA^*|XCKSJq%z|~Nw=Yy zTGeLD)bLL7TH(a+5rNZX`iJZ2;*J;>H#ZAAXzsM(1N&(X%o%IRg0W{8SM@?bltp$c z?hU=JyKLCLUEXep@5+fJk}bUZS*?A}1!IB2*bMC9bReT>-U*_ zONWY`h^0c8GErC7xyHLRw*yBFeKR&g?ECYgOYF1qy$;>{q=9^&4rHEFB!#``!-=>D zUTfC$Uwv7_PR{r8kqer?&1!isE659Y0IwF(BUYzFDQ5ng7>!*aLkWg<>?e6uv{UW# z^!yBnAVdFy+8wg}S zbGLaY*i9iq2yF1;@=I<5@UoUrA6rH!7mkbPfx`LZTGpZ=5)&+n<^K&7;A%z?>4VcW zs?K5;z&o4V8aOI)L_+?+`ZW^+4rwu>obD1z#B#9|V3uqwmg9F&p{qK`T{mCCp!4~t zg`ng*#}`9qJF2aDdNdm+{WQCxBA*6Rw>zZvq2^hcqEmYOv+IhgKYNnq!oS8_YZp`S zBn1)d3K_fp=94t)?|XOmNQ~z9vz|;&X`u&ejV=D_VnC=BJ8wIwCIwA=kfBB$m{ASUzlzq zy9V6Ez^WyqQ{)F?J>Ly zQHK+G)wH^T#{0#tr2HD!-FOO?O}W^NIL*e{K@{8x>0NN6tl`%-BvjBQ#{O~^tD09i zMralRuJBS0af7Z0ZpBOCCx?#=KwuL!ok*M53OuT7GZBdzm}W#g_M@kA!p7;ZAM-F@XPJ5|C8V&a8rW9(4r*i^zbK@(o z0h3(4@87@m*Q@4M2?|zXIi42U>g|%-qgc}0MNwFkP@=M;LnmSbX-4k;vX{WYe_{e~ z!+`x0P42rlvqb5^#acgHQZc?LAQp78O2G0_&%WSjUtBFIqV)#hl-A&%t~gW;#e*W5$FXl8Dj0wLwt%jRl($S-D6fue z?OTJLq##%|X<*D`LgW?SDM3^SU3HOc0pq5Z^Hm9uY1#2|1gbQ9r15p}oj)NV_yAcF z#_ZwUL#kC^dadY{2}kH+eK(WV5S(qG?QvTrg4rMOmB<9`?$qc=Wl2ntfZ^ZiW?1^i z(A7?{S)fZ9cOIH6g^F7-0jlg!%~;66A_42fUI6H&YTf&s5^TCKMs2pbojrW+XgA?v z(^g_$Ma39$qOS&#s0C+Iag4k@KXr8(PQy_u1lKZmL%I=`o>oy_gFan`NeB{bFTU@I z=i++wmY6ZHp!88u%{t@0_1Si6Xmy%k_mkJa&d%^>E1wk?+xFG!|0?Y2bUBr%hn${L z6loK9=h81SC{u2B)~Zov83Hfo(4tNXo$N!BjjOg7GOp^4vm5C`F%fw{;Mq(%Rsq#u zSKALv5sUH?`CqCcV#jld?nVV?z0j6jcR+TXGs`gIPnVW$_1!q3EQ&RRyEpf>0XOm7 zzRfLcLzcR=wDRHNe>5}~@buN~4tULi>0uqh0s5DS2Sohn-JQsSUb{Fr*U2lo)pUA5?{Y|M!rFS7M&H6y~>t~(%v*so73RlF}vTZiv97Qm-9kj z_U;*eQQDZ|C?#dbehFJ~kjvOl!yaU$pIsRR zKWhC2EED{Y6w6uwMXR5s8Z^@sAN3?u-Zn5c%f#9@Y2ZNK`DI0ZH#6`F&M;`Pka;56 z0k6*ecgdqBsL*()#6w-FE#vg?F3gz=dn)aMBNn66?@})5a!m5(RqI{jl}YyOlfuCI zyQc7$ld{s(-IbPwqLbIK=&hl zg?IZ)z>dg&@^V3DMVTDfeOT%DO(MriBu$4r%rCkhvBtL7#?srhy}y77^d+y>>`A$w z0PSy%2 z+nGM6l=%<9WvC?xJ;q)3Z~16@8C>bys#jf9^F`dg)UI2 zA55QmM)I$~AQJrXJMw}Cam+pO0V?FgME8E?Q2u1u0cOY@z9A!|M?1{`Y#70S`rkej zNSV?=g^yyc7uRP`u&O`Jr^5N8f+pxI^h-0Y5~tbRefa)@Mm^|3aB-0!fi_YVQzeC> zEVp(Hm9^%AHN&QwQxW~#;|(QaGhNzgV9JJ#`QR?+)+w*iseCB~zcaE2@#KOln53Hk z!zOs)r=g1!#&Xin^91?(p$m#-f1`Z5CinONgwAWWPLhjN^&iGru1nZ=uQwa zWIdOhRb8`Nu0J}E7D7VOahZ|(W6`X2VGx(w^z}o6rcaS3f;`KvfGV|U3{+ha_zVA4O7mF)K)4iq`lSnhm4Z!9l9DZUKj$y10RJ zzw5w%pyNw)V5ypjwN9)i6BYf_E9d%(=A9CTL(!bj)|G{%o}q4p7FjPw)Ax>phcUXv zt)yY0fY?czr8m)`R5wmF7BrJ$?KCS}jSaYY1KR&#OXy@s+)#OHL$pCN#`I3fm(jv8 zo>40ptvxU8P3;dqfw1shFF*4gw_CutkG$25FYcCMI*o{>&7|&aJ`{t#e0e5<L^PwMh=+;YyKeU4wWwLbWTn|WF z%0VjKd^!nUeF7zKH_t!G;W^qh_U0v+!cf$U+tGhVLXmT<>46oj%L z_q#qB#^7>`w?mA%I%#&)4{kGZFYkE2eaRlZz~gyz7Fi4-C-(NybLWF1jcQr? zAFocWiuHUV?H3c011H4ei1oD$u>0w&mbUuS*>$uWROAW#(J%^P7v9B5BvG2Jn$uk-eVb7WEy+=R%!@&-*ekdhuAZ&OMaE)^C zzJowha>~le;Oqr3cWQ5Mueo`xIb!mI|NWSln6=km<<}bg7!$m9*tqCih;766#^T?A zw?{a8w{|DXACI6K3sw{}Cis;YMm!dq_JI3+fBV#6q;j!i^*{R5V2OGDAF1vVNO<(; zJr&4c`KmixlmzqxQ7dNJ#x?#8-iT_z|Hk}-kxE%cr30TRa*QaNilQ)VajLM ze(mi?Ge7h%Jg+JVnFvmUzz>qiWUo`jf1I|1Udu_4EW!HGrV{ISOYYM04-fIF?TeI; z@y%rkJAB_}$lyZ4zkE<2?&K6zyN%&F{yk~N7#TNYZ;OBwuqV)78V7!qkUBG*&L2vD z`)6!_Ixihj`GWB@;Rm%Y z#H)qT9>O37Y@SmCtCo6!R0v}5PtM$+_Zq53Fog3`x8rJ>7N|-mQop9aXEW&AVAzRc zP6LEi56iwy12ygJmV;RLR<~2ln;=G>Fl-*f8_#idq}g=ya%F@;>iQMf@lSn8bD8OS z3D&Z+?^pc-%VnW;UA4Uzsk0fpP)I=eHAyWDC(eG2M}ZVFIW%s^&4t8T2Z8 z@9WI&^VZB5FU*p|ur_D5pCFa#<=NDwI_7!vbxPLMZ_wkdy)l`k*&4y)8&1yz}DXvhUiKk(#Lph?@~&k2ev$Q>Au>gz#g9 zHyy8?wC#l^v`sM?01$Mfi#tb&ZQ()C{( z7@&>7h5qQV!n7qXd_3(M$!3=l8hQ#=Z){BJe`cB(Fy1re7|d^3EP;<7y+p5M`*|?B zcoF5_gv?{nPzlPn?1@n7+fJfUxH}9F4M=)k5(JWM-QtKYvM@M zxf;jCi#&Uw%HAQ?N2QU=?Sm`$lYee7hkd{-d_K#stCR4YpT5?v;xi6adi-*SyiA)g z&2!d?`H=q-j;{S196Hke)-#!~QqtvBXxnHNnsaaGBCn!?KdSzsOZ3|U?>Noev3lc; zexSl zbY8Yu1Y)rf5hKx7{|_&(x>ScfFQrw5P5q$`!*3LQ{Ag=;$vY^hq?AVP?-z6kh;ZX!`9YTTU!?cB7CGNNrbZ}AArABb10}FVt*L%B`Xq-< zzP`SrF}jqu2Fz+H@n&!ZEXft6Rz#R_dc1=)Iy%}t;x}z~z4_YWM?F!+7kzw2%1Evj z3H<>La=o=^bO@6Y@ycD@wfa7|qlRD2tcNR#Q3g&M_&g;-+^kPKNTD?>%tHuDhd0Bh z;!@KfC{3sLef|B_TxxHLr;2oiq=<$+{!<^{Jk$?Th8CBW-ks-hJQ0Z51)lkcr=aA&(-KXec|y{L zB{H9?9(W-3+1E_141D{1tY6|&7+*r5{H_D17sqR*J|Ca8l~>mc7yM0JLGyz8g-6GK zHvJ}gf#7(*DSqEDf6#L%{Jz*5kDsMO}>;sbH1pAsKea8|;-P4eeV0 z8(1{5BpPLzqlbu!nmGelb?zMf*fC2>wNraQpcuZO;9$)z6~Nk<569*CK_D?PUJyur z=kEM@LEveO81M>@0&wBI2weDMPxnPM*aN@3K2HzgyFcK2Hsky6u0BHrg-rVVFl?#H zcT0y{ySB!Ou8Y=*MWv|&CFwDDeJ?ci@?m_xo{;AQfjEc7Ovj5xE1p)8_E#$KVoB9@ zSH5TVW@N@;P!Rq&s?S*0738Hi%JUXHF!Ql>%;W2(Akd5O^WP3G_X~uWB>C3}pp>@* z%wZIR$;rTY`eNqd7|a>!WZTiH9S^kkT{+)^=Yv=`aq4KPnqcO|qc#ultey)AWS|3) zIaS^0^1V1o_zR})cHJuae~W}~(&YP9yl;c zzi$Iqr7G%?wqPP^SRL=@VwoZQblzlcBJR!=rH%o^qkmcI!7%tbG z^8mi#@4EeHmLAlhyJBGOvqYo}WHT)u`=R|a(n#pfYn?`@3zoD#`bg(Y_xiH>`)e-O zXu;1Ya6d*TLOz!^l2DFZcU5a3N~FyO)05Jga-p zt+x6{E5z2+M)Zt!P1TIH4a1uGr%n6sML`;~ru?o+_ERboZI7W4TE(4?v=B22<3o^YmcYv~C# z5^J+@x(7WyT{)Rf%02XuPewuJ_Og#}!c`gH!nE}-vyC0zMLIw6KI?1F>);;ce3l7UmZ z$`f5r9bA1a&eK2t;kp$!BU(~WG4=!0fSL;{iuBeECEKj|nL`+aHvy)P^rr3nds|Od zy`s8NqjlR@wra(;Es#m|+Rh-!80kE$)c8^+2Lc6oCUSZ)X+|tku;JM$b)bsL zk83Y2?{@iiyq39gMY5wzYj4XjtSh2t7r3(1<8YDA-WDTQ{a1W;UZ<gc^!t?yg^l{mFu6oUdW&+cnWauA3>uwwJtFUd?ti67Ly$=L(kNtI`f!mBu z)`oSt2p=8``2ibm)kiCgE!Y~%!MdT6>V`{~>eq9~3F+AIafT5%d)sQEJ=7AjoOp+V zP}!;C&B*@d+J=p;chkeQuH0?7+-7k73K^~A-?(9u9obelG0O&j{1WQ=hmhQsT;P@o z_A#sYVL;?cj=5e_M1$q}(328&wyznx-lg)2`U=P}Y#IFY zaT+!d^GdJeO@kq_B|lg3Pv(X&a8YJNa@zG0!lK_gniiz4=@jyl zrBeHa2c+>J=5Ay^9j`xcVYI3r%^ zqe-ar>R@Sw+0u$!>izU=Qm5();bm2Tb(lA+#BL3!Z{UWh744662#ZUX>e;P+fjWW5 zXJ&~^4<>Y%X6tQf=z~r*#ZyE@c?&F9*_kb~Hecqg8V9}SZX5H~@bDn6(Y(JZBsDFW z$*g}p(t>6kCkgM72kHIe7thO)?RWP5G?Y&UL! zSfR_2xSsUft>FR5eynaNbmG(XE=(083HJEFkAy`%1u4I$r zI2Sikarz`5NW&GDoT8qSn|&)8BbnwJIst#WsMZWr=47y)P7Y;nSCThJS9?}+uMfY$ z_7tgse-1>N2r{`nKWzS>dS7?#HRS>sYcl`d!rIsc1TI51I7@K%xYzYzA zm$TC998fP&8MRb7F$HTpYD`)bU6`QvMxVU;;;FCUmBpnvhBOFt`)O(7R-6NNn;bYH zD;FoWXi3>(A6Xb) zAs?cQZjPM`{ZR5&FxymQTm z^gFWp?#4Eeni(BRnZL2*Dyb1-2Kl3c4ItL(E6!0NY7(R{Nc#S|5jW&UXwo16J?`z^ zv#3{T&?an*FbMS0<9yhZ`NqMoFTOGIvy?NCZJ$PUu51(;12zk~_x5*@1nl$Pd;j?D zU8zx(Lk=UON_DC#?h|1BdF8)+%3fTPOjoja08m(vtSmIK>JOk9`D@ZjUUh${>`?CA zCjNWC3ub>tiT@_A|1^ei(qjJP(}ugk_HRsEQBa(WIb}3Q0iU%nY(iZFvk_GgIW0P3 z=}Gq#o@ovx?(6w&xv)bwpwK!!?prQC%y4ESd|lwTBVzup2BqjnM#c7}0|UmIAJS`6 zkhkYcQgp}qwXizxbdBqB(YAWV^lZ~`nQ4-ZgPIA3;#;E<60IZfHg4RS8tlA9mk>)e zn*PQb|4LO@LRLT~D(dzCve;I?jz)E8v%{b%Lp2Tks2&>!ER+hdJkn_v;LEMSy_qCBu0zWD z379@wssXre(B!P}0L@3g5YeH>u4iSY8BtO-7hjjsl6mu!bh-Zi;UYcroV9v|3Rz|1eBnab=-%Ox;_8HRq3H_9;8?_GGCNeO0i5Aid{hUi`| zD-M_upAd55p@pItwJIaU257I=CBx-CnKsFe5!5FvC3apPt^s3^^614e3g7`F^>a%G zj00t&Ku8KWi# z?D~=M%s#eDNypj3NyP(5wnjfHP5z0QKrAHeyiSd_>_kh4Y8Bypgvw0kP-RDvbMaG* za!tBIGI0`tsB6H~yW!PtTl-F5?3^6JiF-mF&=&c=w(q2UNj#mhNEwx%vP(n7#kUZg z8vtYr^#vTb`-|ttElYwjt}57RJ!AU{ZgS3hnf=OaL zMsQc0H+G{azn2y2st8x=&L>W~tI%22ZQ16Ar+-Ab4j(fhqBwSqiJ9h3f|WVvs`o!G zF7$NGX>K$XN`GB4T(Oxr+N92EEr9Rsv(A@{3%jm%V^CQUZY#CtRB9Y9Ck1g8@i`Uh zIOhGMzp@Y7o0oII_OL>k&ae&b@QP!wXIb#{(a=P#Q*T^t_2N!4@$~kfJ}q2G11ek1 z#~%V2^eDm zWv-5<`sQV)*vf$o1`{77r!{%D&<_2z5dE2GDT)km(o>Zo{Dpm|_S`P>(JHciA0A8K znWX`1d|+WBZ=iMG9T0dE)Z(tL^5MtWY$x1ZA&uyE@ErQ?D8HE4xYusbEiljRl+qpC z`xEcve>3u29*mrGFb`gAex9NNOb~!)|GonLRnAR;=SQDw_|0Mw>qiiE3QKZPO(AEa z%c_qaYT<0@GRQ>sMiZybTv`j!_43^U;>2flJWa%AZv7LVIpTbrT_cmdz8!gCz5;8Z zc|Mz(@Gw;1>;RCU`qVh7>bbRlarM_9+bV-@Sg|)jAPE^UBW`LO`qqoa8~k~tO!59P*WA2m>#}nXp`&Nb7e<{RNj<#&}*jLtLDXKMh;eLr=4c(>(|4ryK^y-1M>VQ>lHDN6v^W^QHCI6^O@l$-e^XM0NM-n}b1_dbU$&WB zqsZUOmEC;JA&-LzaENLh9)wf-_Va-ERHArb8C#w{DQPROJkBvgi|zX45R%u1W8oC7~_#;tmVU9(Z{ZBM9gX=NNe^ z@V**Byv&o;Sf`%IOi0(^49UJ{#6a=N4I#HF2V39Z zkLy@qWmIH6&@Qtx0}_ai0BV)3ZoVlrw=HwARH<1|76j7#F6e$%10sFhZ@AFMxo`ZW z?u?X54$;R)C@4-ayU;-I-KmqMmS65yi6T;KEO9W~^;Ov9B^xSOpVb8VTsHUM8!s# zBXw_ra)O%BL6%?ejRqc&iJl(m92O?{_@kv`d5NiJPaM8~Z4UOxCOe1A75N}^o zVl+Y9EhP;SG*sE19V|up{kL-WTcD1$4|DiD4r}#jJ+Wb===*U6{%Yi zNT3tE%}x##!CAac$D}&5cOSl%@B)R|Fv`b_1&S-*T4-+z>PCYfZBQ zSM0wMXi4vvXWJEPYN(^QXl!X?*%>M7M{S2$w6o5#ue!dh=?}?oCs_+%qK|#T~L|&bwgZpIPC9Z{=d^uam zJf0qHGn>B^)PdO3EqA6yyW1==)KdM2OjgG+d8PHik*p6W*XL1nMq|X@5G%WthXdbG zpT4pBYE00rya2isyB5<9CK+a)9i%6r0$tY4-c@pJd@qH6dWgw1Z5X*3;T?^oe-oGB z>8VggR$QA|pBoN*K@st^*vB1yOz`1X&VT5Eu9Fnvh7^%#&l*wGD_o1f%LC9U4V=1u z)Pz1QNSYzz+#If7bK_u6>5`!X1K*7dKT5XCgzAfgeuQ(SHCwb}sj^NreP$@@cUHTd zS2BpAF9}}))Eb^Z<}~B3LXbSkP{b@G+RN6lcEHu&iNR66A!;&mx8IQYb*iecKZJ_U zp~!~Hc^bw=ADdds)m?t7yD?W+M(>SFlszttVoB-Gca!sS+6mO zDY332T>f)~`^6p|%5ck|ry~wzpIQ{@^0}?|55Zf);J<_SwP7JEqY^-kmQVaQ5_QZy z-sVE`;%gh(As~jYGO!gJGQKE(@-I6*x`W!V5af;!G`Yd0vIYXWJ--vTC3fQW?n5!D zT>*2Nh?P2~qNw0lt$$gG`uS0Il?izv$1faH(E$bj*igw*#wpd7Wp$BBI`96bY)c45 zJ1$l%N-suz+X?ARZ5|N~`LbQdO!&<-;y~`oGb8Nqfg^J!^*{Vhr7QFAf`pue-)#>s z#ltDvJm+4=+;#ZnAS}W(6@lDgE{5wWW-B+%RiLQ7FA$xW<$$fZ$a;)V9^BJDJq?$P z89@2D$o}|tmbaIUZ|tyeq|Q%`g!wPp%Hn(g9n-0VOgqEdzr2L+Nf<#vk_*lxg$eIp z#dreZ$QcOS*25>bUYfpx|g1YiCOX{_< zmZ)l(m^}hJpkSeBmHRN^Iu?6vk9WG(#cyZ+=nYYH4t+ip(s}b2hODeQ-BWlpGn%aI zCY3jaNV!2VDtr>xPa0!;-t$QWy?!6_T2HmghomymZzA(mThAh25f#u{`1<9STe%&F zk0s%lui$6AfA%zMwnvDBzv$EUM?|Rgddr^=V_oXoo9dC&TT`CwQ?2ovrjt}gw~{5% zh?@KSGjS%nuL`VWe3t6iO+q(^j)z==F|>GCiAX(JKd z!?opQw$xs+3oze|vn&eXs~I+;9V(BTPY0W*|DoNoMkyRaFizH3$n@HU#VG0S?BFa< zP6n~mM6-FplSJFloqE%^NnX4}{pgEwj1wpaX^e*J}Y*-goM*xamRb+Jnv)kb9A( zh9EIp7te!0l)Skm=-z%GpeRu6{)@ns|9|K?zS!6#Y-uzd18ji3=WST+PyZnJKY9eu zpLfj%=Oeq}=0H0BU5u~UDtfkYb-6P7j{4)R)&pq0gyx(GJ-SUY@OlexGkSdxvB1Qx zhi$SZQd+Le*VTGIIcBYVpuZ+{Vq=~g%L}kOk9(u@Sh$D5rZZUrz4a-ejaAzKc??AF zmTNSV7tEh?`V*Cn%exmIsz;xVri*SLk6vZ?J=Gqrk**ou3hncz0EA1eoCzD~c}wDw z?l84`)1So|`HXJXt8$@)T9yzXOPrA(S)EiLl_YN_$R&0gF5Wa)grWE2@R#xYaXnh9 zZzHlvaEl9G6u9mhQ~Vb}zy(PiL(iY^?Mr`7#v1@5v|C&(`bUCm5{;YRHojY(EbehS z`0buMgl)BkyX|6aWkOxUrl7(SRbW)!-{)(7UYPrswmM6nmlLDL(>K_+NWP%kJQrm;JiT5LPuOW) zb9;aDj@6r6a%i8oJf_9ml5SbN|EV`Gul3Q{W_~jfuU`Txac6hN!~~@9#EJCEt?$!y z>PoEDdsyzwY8H9>ijq_uSLD(?xgB*a!3wp;Ag3g&O}AjR2Zs;s>$?bm z+WAjE_Ve?C&fol-s|*VM;FvrDjQCeq`Cm2`n>EWQOh*|2INh$%@nQw#_NAZYB>%+E zlipO3!IEw4a3CPq@W(#fx4(e3cdMkyeUwi3_uGg9ywh=atfbn=etI|UY1Df<7Ohip zE1b}`+@;A|OnUkO$Zz`hx2ZL`Q5FH``7|Jb&7s1^8N8J*e&%VT9$Ww_cwN+4J-I8` z{xATC60%}0#AikV)!j?cJ&1mIjng4VJG>8;LCD_UrO>`(qJ2X9zJ-YR>L+6Qkjg>4kmoOKG$yO_|2yI(D`g@IT9r0U!TM$NzaU{aH|h9`lRL?M`v!+5(O; zQp=9WS*}trCQs6nGB~f8?!GgnrKP0m^a`p8{D|%8Dm)2ZxhXlJ1O59a{bRyi(sSlE1(U&}@*}-lSzXZygH_=DK_qnws8qq;=n!>L$;@ zrzFD>PG9Yqz%7u>@8@p{FYdIg&qsdNcAfGy) zb~9LCv=RgGH)y~?$jB@_xo1#qeI!1u=t2M6G4_%g-MsjG>F=^^+0P{%X#Y`s)Ley! z4jYE+9x~e56vHiP_Iuw-*ZEf1P|ePPUx1s3CnLvrU%%YnTybHdv2p6l=-_glGLOxu z@Wu4%+0t8&q;!4ST|ciM$undNYk^s>d>C$X2m7@2$FGa0oQgl7?=aZ-{8>K{#A zyrtr1;iC?R=LJWat<_g>-JGACv$KA7;2pi7&?<7KPc)Q8O?tv7D44+iU_(aWG&||; z=odSgpTQ;g0D^}?x5k$lQ}6gxyd1`qLC$sHg7vTTQ&R<7&|1mZXjT>wOr9Xz8FvsGjQ2=P8z_W8vCer;68~UT2KY8K)CLQ^v5@k0ah3H0gmks3bMWoO=)KtOz*)wu{no^uZcV4$X5!{Q*ZDSV=c z=6#sm>)FVu$&?psyb3Q>dEWEVOyaW+gW(JiO8^uVrC zo1<)0EcJxlsM*vd=P_Z&nlS`Zm)>!oLB~}$HTwK~)@3A_(YuB~h}m!3c||q^b_(Bq z^piO~W>scinnTGo3jZe4N6d1;>(WhVk-I3S{INZ#f&l$@r9v`U1UB)d{^yZT>^&-V z!QZLda&Lu0%wV>M3p0(~vJnvslx#J0C8#@&F^?ft=*fwR{l*`jf?jOxeZ9V^tO9&MeNb(x6zbH|G>k~pZ1I7tGX*XOLvX}PdQ?WCpB@CCKUuqt zWaZ5m40VOlmWrH`l{n1E{4sW`;#pBR7|`fLb26Sf58?=CYpO_a56sXvaFSBtW-5c3 z$#}`U>Qx+K&vIQK`Chn;tn2MVV(unxJRaG&|EjXv<5p;#kzD5RrZxA(wYiVWCc7R$ z@nHoaf=c8hIO%KkD1XSPPJcYp+UIjaU`D&}kQk4%rCuou75}oL<*I)>xB|&KRV%E+ z^`DEznMMV1MpNTR9!TXXaqgv2*dzx=L_X_1k>cpJk?ln8ZhXm#Qr};n227kzCuzpg zBcl`o6WDcDfNk$vFreS0n>c5wkD{9=d7CS`cPAe@nIH*hL2KEW1Ddh&AXhBB)@{I` zJ5GTik+&Qv2=3lz+OhnVF>e9S9G!-taz_s{h?NyI5mojyLDkiQCK9l1#=6uNmRt$Z zgA{tsZmxMfS1{Eb#>X6Omb`_S7ivA7k6LDWq_Y~Y2*rgpzP5^njL3DpNeNC;YCQ@Z zU*Tk$@-QW`rTxt2`Y-d43}{a_UWTS;y})zL4UJzcxrL_(r-jYIi(peh#@eAzANN9% zSCrEgmTLVnEwHfjkX%us|4|ChTn~TWulg;!h z=={3UaYTa=DTrKuZ_|~plubc6F4#2ioxg@oy$(cN{{P|A{~UV*t<~|am6Qh8^*Wcz z50F>@FMzWe_IyMm!h`qX7k=AWZ0>%4+Vt##dpq5Z?<1>Q?l!f;ZNMA&MPn`ISMx*t z@y#{;XQHc{OWm!hr@E81EY5^F`Hqs~hV75hpiH2`mw>VbroeVl@mTv5GIx8V+y98b z$W{ZX=Mx?d*cm8jYg*c5_gFq~7F0U3lxZqtYbp-B$2qzKAvyxp^H-@d60x5%_L6qg zQg~v-pPk!XPAalM`ucR~2(<$bc$+)-f5?_s_FVUxK`{wzGk{><-|SKWK<%FnZUJBa z-Nx4W)6*Z9YsU~|oBL1efm~uQ4=6_CB5=U#Ul&{c*v{4+eO%$sq0!@M=U`twNH-CU z-^rxA^Z)@dqdw&JOq1K$^FwUyoyOf|#smjB@nakU9!q0C-Cf~NJ9KN1m8DnmZE7jg zsI-NE4=SbtQk^mt=rxbK{V&SFcet&3y^?p_ZKz_hE{{bQ9ckt;M^MWzrB3B!lA%68 zLH^jwSNN&CAJn>e{pCCod&?&%?I~`K-R5) z`L7@P$DdmWnhg|>lL?*?KR#c%p#=ivpVK4#{J0Es7VzGdNo>Fw8z4;aHlJ%bWIYAI zv*4dfhL}l;nA0TifBakN;$Q!y7CYMiF{k1P{7O2pRn4IzsgKq>m87%-O!|ACF?WGL ze+a~GFPGEDDaP!DRcf2u;n@=RK_sX?MHm(k= z<`{22U;R;9c}Ti=D3KSwJHF2J5rY6`)ea3J3yKey;pbezdf$TUzZfBXEIzwp;TBVl zwdEZP>lm1?ERtLAO9l3ldEE1?wt7(u9pu%f)e*&%kBVrHq-M(*K#yT6+ZvZ{OeJ0t zIdt+2uIjMV{Q8PDl{DE|LI*rJ=9h8QKP6zcH)r#gh0%T9(8OP5x8`7k zq-VIv0J(5L755So%QawM+iZmy%UPi(sLvhG|6bJd64O)tVqsj}V$J#D7*ONMs-jK= zjq9@6&WBA(iDrNkc0GXGN*0RSeerB}x7D*X6)JB9j#RqG-lfuyhM$h!P9dMU`n7kt z%(c(1mBljQS4Byj_i-vKfcIo$UY@S$%p^>yoz%v?BtgbR7Y5`3c|LG@b+vcq^XI8j z<9BOw5HtPg?=ZC9*7W@L^*s_&YX@UyD^K?W7ROC~ENqNo(|{9DONEJhcYR3c7AOER z4T)6uUjjjCM`M#5b>Mv|JK0%0DDc&qY90d*Gg1@;Y_?p>LnkBi zlxt+9K&E&GI&7k;U>-nGVa*a1kqm(WBF`nH*stf=Yxn)!uh;MTqZeIX*Z2GReva?Y z;kxjx=lvQx}_2C z&95ZX*s`VT_joPL*zkPsgUlNFFN-d>{>#E7bMec^-}$AsZ}-tp}@8{vtw*0ju z|NV|Jzr#NqxOUEdy#E|};JuI~8{B;}ez{~n^66mD2Nw%}99fk-k*6Tt^C@8wRVlvB z3CE9jE04+Kh(bked}5;TD}WT>dieRSUq8_QJmC}@tj)f75kXp@zhRkxLOES9+N1y9 zPn1vJ*ZCZpMTC zHj?of_lw1+@1E!P$hQovAh7cGSuayirSxh0ql4c^GqT%Kw{*w3XRgkz)9tYmJj;|J zw#B-<4|aZin^Mh-^RZdru2GbW9gr?vqf9q&)JVh7b8qJI7;eX486Kf4~ zpZ9p&F9#oRgWn>QXWZur^Pqz_P8(y!l#TKy-U@$IAa^{ymRuo|OTzg&DqO>6rM^|Kb_cg@V^Q~YZ3_z+Ox!v+ z(4arv6Hl+W&2t1#{Oh)!I)gmP1FenlDzWzyqt*c3-wcFXFI9gwxOA|Vk$mER?)k1> zy?f^$Y1&~s+eg`|t1il`o0^NJM~YxZKWBb2crO!Mx3kjxkgO>FR?uXgynT9*BQ6)& z`PmKN4y;jVy+W3ujl@oR#GE&sw zu~4*3)`x((lu@;bf{KjJm|fYZ5eczq(X0MRY$RrhK@H|?XO8|R6Zpy@SMddm)@2Fo zV`I`O`^ow~CdcAabb*gn-7h$dyIa&)6=}(AOTZCeXE-KYcdpSeY0XHFCXtp=I4JwIr zr!hbV_2~U>@yQRvEh7H#EGlMZ4%8k;@Arm-H~91+Mh1uIj8LRE+twV~cNsN?p%xyo zuhbsO#Md$t#0f-sol_zf4@IU|AxYCBv68QnMv)V~BPI~L(7q#azuZtyL&1?gLLZ8# zrAOokqnV)tMUpf)yUcG?Qy~)`Atum$_YFZ^(ye3;WEq3eiYX!#QT)_v5>?t6byC6uwWe{npGk_6LPixBE_!#LhC6DfydeP@tc>H6b|23> zP%?6B0rK z)Tbe%ZPO{57M5M%rs42v#xVmi1W1Wo4dvp?#Sgkr1F960J{*BV@U{8&dR-wklj6%I z1oJnewCDp82??GYyG=G_f@)y&%~J{STOPyXq5Yy(#YYT5-6GB>yUZ(XQe|%S3tiD~ zd5$B1qqcYnTeWayE*Ml+wexGxO50h>YqH2N{jS{2Xwb;7C8e62DtliO83^*TdrJ+) z^}67w^HMX>8o0{3Mn=S&W)0Ya{@us=`~SZ|Z_aS%7JB#2e;7~yW}&&wfTf2YzI++J z&C2ScW}WUx26xLJXgzPky7J=1i(i{Xyb`KNoGD+VduA>a{$%vH%n`A@Afk?XQ>l-g z25Y{&`^I9JvHZ$Gzpvb0f7^JHG-xb)(j#3SgC`1e)k(UQ48E*m&*J9*kNvG)|M}-G zME7LNA)8Iv8VNOfts0%J!KABjQH%c!6nl5{OIPS#jbdN>D|a?>0g>PV8T!MiEz;8h z$+0~B6Be-9xN~`TO@tcO7enK0(7iiP$%7X^|NZymD(}_b6OCvhX%;Pu$2`JbGi0uN z&|r>T7OiJJYTUGZ)SZ61$KwH_Woy4?7yZ= z+xy!HR)4&HZGG_c>C+v@o&<#`$Q_DubRm(!;L7=<_MQBPzT9*oo8c{L=Wr`JB zuy)}?r;Tqg-I(rHqI1cqLQ*o()X&X@f@#=-!3}MkVw3Nn;O!%u*Lv&Ux7f+<(?gR_FCj8;1V}zZ| zp7@SFM1?F4_62QQH;Mt5)%w>rP88qtt7k-xBI@Kd+cXUY^+DHF4G*qkqiAcMmveno zm}>c7V=pmCqFVG4v7f~7+9JaSq5D08ka!i^kEGmyW(C- z{9#6m=3MvQ#V3Cri!P%L#*cY+5)zG93=ogPn4?{$HiX8*ltBRA-9k{^t)wT39+lqQ zUR6AeY(+rsVNvOos$H;>LxH*A0F2qzoMf&FQ6W$H!qyCp@wAdP5DPB&!WxFt zx4b^Qf#xhgqXi^Q;E0}5kDwJCRetbSHnBt{w?&v=tKDYH1*qj}yrsp}jL7DX-<$It zO&pu0KD8XP@IJ(F){enwe#S9zZL4zVUJe&}*iz&!;EvSx3Tyb)^vb$L)GqtdoG(2B zzzp|HQL!{Gn&jlfwE5EQRwC&)zV&W2Bd!S1n*Nc5TPV3-kg(9;Rvu&;?QsgJEJ^ey zN9?f@*G?*jcnin^QELfR{6{rQ;h;6nY~jN^s8<|%T2!?;$z3W{J@sZ1$vJy6O&D`g zB+q zbf{Y6B>_)-%0*=1iy4w0VST}$F_UQt&FH2z@T^l_C&m*V3rCXsen%P4nfoPD&K?2I@k(+Q74U_-o+YrkxzvdNh*6+z)4&CRp=KS0n)~&F`UFU4bbM- zb+eC?Qn14>5uR|#5&)8Qsdf$Y2@VIuePY0@viU)ArL8+i`Z0*T5hPj+y6^dxYr6|1 zQJ2JB8{V_9P6e#&msu<_W83~HAyOXp`GS$pudW2`!hwqH7J%jy0^n?FPc4v$rn2)u z29td^j#&{t2oVc1wu* z1#|M_pbWRcoa?jc=ZOa0u5+8NO|V%O5k@Oux=bFS?WxWP@wUHhdC=3-bDyb;yoOp` zBIj{55|P49;Uupqmm)`%Gw&N%UU_MEQh`H_Rp)3sZb@qEr3((Wp;zfDi~whj?k~^1 zMhw2za&3!aKm?!Nq>hf0DRo)p29Q?Iw;Kvde5@`aR5VC9qupv7jH1+5JH=+VZMp`Y2}K4k>F=}rart@9 zmURK5@5zH5euo>270;mP>(eymG~I0wp$pM_Zr(e%lUmi&ZgjgC>=8<-NEBC=GvY$f z9;Acz#~D@$ftG{eBhYKqp^-qYF7ck@*sn~Z$7ilQx363iwC3p)4;h5$9xe^Cj2qLG z$5&Sq$77?i^*!zpc}BY|GQ&}xq{DuW2>1vT zeXVUJIj4CuVkU<-c!!xhIyRGz$&F%0^bJMAIEn`@YJ(b}BKNR9?>9nn0sf zM;eDgmy`a598)G{Q!u!IA{>paz_>8fl2EwM)9o{*5QErv=+VQKix;Iw7=J*#FncZc zsK`5d>W+^)lMw6s?)`4Jv-8_KrZe=6A>S&`m;ki}e|m%t{m%!z=c4a({vcDk&E9*! zgQi{Hp#vCmTn9YQ%G@7ZyVmGRGIQ!QA0^K01p@tKa&GmEb&>P{h2`N|8hN)Sbk@?Iw%XeKwDvZC{8Won`U_j07xlAHJI)g7oZ|$r~m+6aq2I@@Co%-D7t4wZKU)!c# zXE1=ak1CA|ki%@NgaUU(Um(}63K9^vU$?P=wIgp}uyVry=dKNw3<)8XXEbNDqKe~Cfkoq(w zXh@4Yih_Kh;fD_Jh7flYSi9Iv#Na~}m4me-?+C6ekv~y`=b}BJMPJ^(3bAadtfyzY zDd77T^}2rkb37^NVMCB2AqyipX*%fBX!O_6r=mYD#4q2-Z0Sj^&k>#>sih7}q)QEO zB)53+b9xi4z~WM@b!eMjfenAPUfOwJdF8s?XGveU!N!g56)XJfs|I_dnPP@-V@vtq zNyT3s!j?wZr=A*Gx+-)urN~hWli^6p_ECHPdxG=MR)<_dV3bHiNF-6|EKBykrxlJ% zI7RzI;cLh-fp{}3Dt{xhDwKqc6Vn=~inBZkiN?UMFx^aWDY{-*?cSCL1>k0KAB=$? zWqOOII*JfOs;TNpbW<1Gl)z<|+rAHnY=&ykecmE>v8+>_tFVzozGY*77*m4bgsnNJ zyziM0@m9GYNiE(^A7pE2?veMZ9LesQ>00iO-i#7oEvIk=t?O>ky&zclFI*@a79#9Y zI#adKAUu=vbI@0v<#P32(O_M*q6y=a%6)C-kVsd&ZXn6{P<2hMgwmCA1Km(LJ?)|Z ze(+-n0(Xj(M3azZnz}O)&-X+1?o{97lwoR+y!*Rng^l3Gi^3b&kWNjvYF)bW2jL;& zM6jyvdSQq&($9QHa6=K;%djkz4J6E_f$52RGD-dkW1bIu6H}Ix2K-RMM?Hz)KHRBj zW{_AO9V2WSPcW;0LXfG~Igv|P44&9}au=yB{p-3#sAvWoBj5}?ke(cq#fAr>i@rb< z_%JUjnw*8dbC^`crG#K%hs2+jLWc%0s^x*bDxo5n?+)k@r_Omx_X@pS`dDi4TzLs3+?!uK_K?2`4$I$%vu84#n@%H;8n7b#u!+8^=$p{ABDcIc*Lkx65Gf8 z+0UTW4Rfg00LSL=@4JQIx#0be;%8cH753~5q`k5!8#jC#b+Nn7Ojv&?TRkB4&}fDt zLjW&#b<^g}n@!x%CRU$Xr6xQ=-7(!BJP&*VrJ%{e9p8lgursBA%0^-eG1BawiM38? z4zUemp(#D(E@2JP3H?#{osF^{p`bnCdxcO@An7R*{V3cmkRH$`Yj*_873y+!X1Vz5 zGt-_B^S98}ozsoiZbpEPtlo>>iNmE^pUAM2J~Ea!wf9D8{g*%l6NlH9)e96r4~4m0 zbN;Nia1rRzY&!?2L=j*l(-~Y`abJgvp|%$8;aD>?pD6YdIl5D|U@@{wz0Q8 zaeYvC7#M0f`0et$= z7nS^o#$TYn&x6?`w8hW;+-x2t>HPhrgM`V1>xGd= zoNf~ZAuyq$9PCQqog9%zQ(XcmFRxqxgP6hGE>S)T=Vo&d#!fE>e@p;MF5n(SBXqr%%NfiRPbv1&W%ZAg`-px#^GUr`hBib zr~mo&^F)GbvGCXo&I*O!$s|oI-n!O_tY}-Cn>KQ<(L3U4&!=_LTXuOp0*nu&=V)JiZoOc35htUIWf#WariG zi{N*fGrn6UJzI8KINZbYaR<(D&mpX#J#_Iwk@O^tRLqC3CWi-#(svFmD#Tq2!C$|Z z979)V)>RhHD`$s^!RP92Rr2sPrg`yrPxxwM8ih(-(&|8lvu&g{-@q*iuZ??%+=FYK zZlFgP9121#eUuUx8W=|~PXsdh#_afE<$g6gy;jwo^vHgpA@6Fs312xL(baaWB6=*j z(X3h`6gq~$#n#+pbS;b8Rzwu{qMRCyJEm$1I^EzTBT_9OxbMaiSObLwUig^u&ba+H zOaV2!r4_uCYW?H+!fJWSr+{h>)=k!3+h?OIG_mcf%&!-EAt-4G*^@wSFG|hDPIqIo zWU4@zh89T{K4};FN!Yi8hk&jOVi?ZHIa_9t2RS#0>WEUB&(=pZQQhL zoPIr5-%l+(t`y-U?4fL|#I07=qdqN~*nb);*>vAJR`HJQlcm@y@?5*CG^gI4Ilu&eJHiw3!mi*mJ6BT#1cqsA^*kFMt9))%S&{095#39eg@ zNI{o!-d{?AD3?jttFswe0X=b&jXH9^FpfM3 z>(0;Tk(nG-X*byo4rAE{51Ezd=yO$lxbpMn%(cxF?5pOf*H^Q(^{&s;rZg9=M7H_^ zG(BB-j>}iF(xrV2*LVOwq5H`{XHJc+U(M9 z93ni`v+bv~sD2rHMGOO~0gO+#T2lD2;&atm97c0BTG ztKH3#xHJt2>G!JjYg05p!l=YfuZm%coZptGOnNdz5V=|{fu2NLoF)ASy#D(nR1~S9%7&R5!E@tut-eR6A<5V44gdikU0*&>TvWR%^~8A@L={9|+V435(sJE(BTlF{y}LE9z7?;gPYkV~I_jPWg>9U5 zFDD%AY!_V0g)blXM(d|L<~#rZN58ZNqbdxaT@jj9_#xj==S>xvS_QFRA6-G}oTCerv zbM2})Cr3;!AR>%e5NEwk!SX(GxjGw50OUsM-qep0*JG&f3CYU4Ll!y$-h$#sDM2?p zt=H&ZHvsULHcNqGN~e1!tvvTM74|YnJMAVcO@M^55yLe%Z7tBB7U>UTD1bAH?YNo0sYH0vW4_fj+SDLLi|7SA)_ayxH zGS^dP5cRt%JV&DuvSQL|O3YBDRcjPlb%@r>$By5U5IX?0h;@D7CbkTvwNV){UZ%8o~2w#z3}m-FoNhIJ=5Cfuwf2)=k)}K-6n- zA^T27uv{;AT;9OpVY!Ngs3}*+{$A&=-D>2H+Y^}6?Srj~B+r|;t2Rh9xF}Sp?n-#P zYZMU#MKAUzDd}CPD!H)8Xg`pe&GvvT`r0g7n_3@Xaxrc;A2H4FKPIN~l0OLh%xPNa z$Wkp80Wg!F-eC18(fXDhRzAq{RzaPkJrlIaDL7$GYq2s|RYMT6yAmSC6N+hS$)={5Et*%D2I`I7 zHmfSidFylyaQE;V%iGS~C=IIG8H^&Xx2@7b3vj0@ufrn9Cw(n_Vv#{$l^Px%wbhp0 zSQs}7Ao_|)=@*$?>)@8DfI3KeR(7zpbxA}%=ca_pApo5Kg<9C<;g)-}FEHdfN?WK# z{1Cr<9y7w~!RNTS03WtKI?8hWRE-u-W|(YS4ZGjoI29|yR?%-a;Gw6f6vVK!sjjPW z%4g%Mp}hShQlvW@2|k6PeUjw4XkI?k8w-@#+a7V?tHw(S_o*2c8`2KOtm@g@7-dDZ zCAniV;JH8Er^{MX?3^Jb3+9!=%be0)Id;Z{3<=sZ9=b8CGPz5^b<%Qq{C1%ypC~<> zz=|8(T`zl;9}Gu((J0>H?{+;&XZ_CBju zOPHM4yO=f@RV}Db;v@_7J8pzjzQaHYJJ)oPYRvGjA8C9tT^?Ij0BqlB_1p zZ*1l*X?3c!1hjC!JrKL*eCpoikFT{fwfqMCfl8hk8;b#navOoCfb#nrV-8z6-tE5J zg}S);TtRpJSS1JdaHqcuNCX(3L8HSi+Adw6=zClHaB9K*ZGdp>bksByPNl4J7uHbtuUa zX{IeXeW)H7z8Qyv1&O0*?O8n1wBvIcbLxqGv`pep7-vpRD%f)YWJB8bHRa$GF8D0d zjG;!Ttv|Hs%woIkz5iwT>hRHcxM?(Fq*+UzX4q)%XDRIzD9qSjUtk2x zi2Li*Q#}fVTVHV(=B6YjJR0RyHU6L=fyk)=tMYrF?y`=nCUG3g^Tn;6TUKHqtk=2?Hg>y%ZgC>@gKCyQ9fQ#%B^w1b`Q+Xze(c*BFcf+?M+w*KU%W#lL8YBUsb!5^YkYQB>Otb3!X5r6%#@#TkXaam$UKIuDwX_`i+DlT-0I=v+jTGtfdK!7PdDg$1raW20uq|BZiaIziyGOv5*Ix?sE-nnK} z7`GCO3&pX!q+I?pe{^8>z%y&A050w)+d2n8LM*8qm^l9{8jD2b^zGp8nBfFZ3m8AhA->*z&O$hz(0Mjb3mACx&PPTFS_> zU}u@K`Y1!)P(CK0Xo7CH$q+HPPv(Nf<=qFW3LK7#Qrt0{t4y~)gmRQ5&R2`111XAN zE=Ntpatq;5c9nJZ6y}$b$J5_NccI0d+5U6roOn&jiYBZ-( zT~G}dG)I(}HEyOU>I;}dh_S%01rV3AG-t}Ng2n4pUz~XGyMpKjUN%$dXra23*w1~6 za0o?x{O|dLxA7C*qf;5PGP9sXuEc5vvp>Il@Ww{u^_(o%>O?xf4Wefep z2}rTTYUtU)XRYbpqTYU|2k2znL~^*PXsSIS$5GVljkAJ@C12271n%Z`a)+IAdkG|F zmjJC)%Y3KFM@oS*ohB@%h#CH7KI`H6etV~SM9!v9J`wu6Exu*6_a{jW0I9kP7d>=b z_7L~||7dc4iEl>6kTd85(iwl~Usl!9PrV;jIln^tAaLaC*_h3o;jPer31jPN9BwgghKz3fv-sv$1RL`J28$>J^KkXxJLx=OyTqjm!_2DS zz?O;CN^faYv=ax$_T~oE1zo?HPEw#nqLvQLad@Dj1Lp%@XQeUo9(7 zT->-7P?4ZtI$(DN*xFFD2ZpXHs78wLpML%Jj_(LCe&m=we#^WM=Z2G%yCBCVNZi(S zs3593jwoNJLc^IKkWD(Fr3nwf2(9CL{=36+X_bDm{(>t{5 zb;=3W#CIB?R%1x8nk+WuSaYGIv%mPd-{Ul%lA!`fy_Ma(yFhQ9KjdaL*omUG$p~^y zCdN5~#-b`h#6prX_MJrzcTK?)m=8?H`|f4tp3My*<|l)}#KUX_RZ%2^YGa;H4`H37 zV=$*=$#8mhfQxK3aD+Km07PxDUqES&X%j6_SX}TIEilrA^4x|i19(rRR43MF6i9#> z2)wer<*)p@;1x7BQS2?M;WtK-)Rx`OZa6QoS`<#Z+g;fEm)-gdQnIBDkck?)v_oeG z$HrL4t^@|=!fyNabdScVLu4P#mc53F^O?ygUrFfK#T?q{9M ze`GB^iC69*nFhqTh(h|r1l)a=p;>bMsYUB=Ms!ie!_aJjrWOPKG1uV-vsJPUm22T9 z04zUE(o7H6o1c$3Z)MAeOIjg`B?HBY#DoGjw#{S>&uImoH`0S*1W+cNmp0}at(Cmb z-^>d|*V zX)5PB$mp;$GkeAIEHHK$%W@$ru2F5{|N;qzi)4+qX$2NaQWQ{9A$ zc5H~+vyrc&bxEU!(Jkc?8dZP>`l#6uL5EX5T=HoVZ-o#1PZnk>uxKqx)8MbFDHr*V zC|(CQ`im7MJjrGL5+TO?M*W>oxTv>SfhSm}9HMbe`P`%e8Vl;HQtU``f_DEx74$mPZiF^wryOGXT zue)>vybXS8vm~vsb2=E9G`x>zi|4BwC`n4!D=ef zq)U~h=D>c9-kSE~8ER--*2e38aeg~}Tp8%>$-uJ-7`kbI4O&#Rs4`u#Tht;S`kgaQ z=~fJo!Wv~)9BPi@lAM1+&Ls(e*KXcujy*#84e40Yjx04-n>JDV275$bvnJDI7fFhW zguRUF8dfw-kcqsN!(I`}4Q0o%O(pL)KI5npEC;LbjL5CB!=be53SpWI)oMo{PM?)J z;C`Z?a9yUOKFU4UThaMPkEGue-5$=Z6i1@nY3oXs84znA4t+fQFx9^QC5sL*f||<5<-gLSIXCwVhT=uEiLU9 zQiW+rOXkEKkT~ZhBbS3ytyf+!S$1OA0?zvx`^EdxHMc@`Yy*i_s+QmAduxs#w(?Hk z=K^Z2wJMPfz=JhJ8ZeKxcd?0Txgpo`E!gc>FE4_ofs~g`zw5pF<$EVEAU1MxQ6o5& zy-c;-VvY#l$8VnI8EZkWraGLv#+GPe@)!J@F804x%6WI8cc&(ONA7%{)H3UtFNk^~ zM@0HDo%uo5V}EWMsTcoc&!IaTXX4IVvG1hgU56gn=O>A3himaQl4-O9PziLWn(AtC zh9I^f?Nt&mBpOL;0tSv+b57xNBKKu!6#OhWks&P9*MmWC&ofozwAOJbc8lOI#cByv zf`)Bv^L;()wE=nej~;f@{n0?e1A=ba zX|k|9@r&CYjsEw#Qh1Vd2+g0_0f7gO$IYvFiYh-G4p?^$d`dMOcj)|faV`Z*BJP(L4mnr z&6FH{N8m+Mesped|Us46P){)v>t!i_OD@^ z&UkXXEjLNmcAPuo3@y|v7rtqZ-vLw#OwthMI$8~x37}BDep_qZs9Je=l&K?DQ4`8O zu6A~hyw&Zv^!H5I%4>Xu`u^yg{@p*fHZl@PzVhPFKU2Ei43q2?JKIid-^Llr zsrQ2G=s0ogt(lkq>$jI(?}5JbhJTOerRh|BZU-O`wcF)>r_vYw4JP#Na2|k_`(5uhSIwG3sx#`T_YuvpW~!MU(|$|+4cE=Vm{%8OV_&z+X77Oh1~xZ+ zzVO@%$Qid%z29)%`H@A#OmkNC+ij>m=L4VaqH_mdOC_4#sb><*7eEU+Bo;D#fCV)1DimffMn}Ke8ZIBZ zX?Zn*=uuECte;l8i9ZH^yJfpGx}L%jJ{wQCxnH*su+?*7XGrUkScuB^)~j;G%c<$K zUD_2^BQqsO;L`8(>WQQ&PNP~PwKT8E2`EBa9{3B7)4ZIL!YL)x*3WXk)<3*A^?pW? zs=261qGiRs0BCPGR`hCmnX>;AtJi7f)1`$ogWW(91audZ#LXc|xWc39Af=c7StCXbw}7E>VvS`6SsEb+)eEgQ;Z?8WKlq! zJc^k7wL(MuM(2c9E_iGb+kXHnOY<7S7o|)I-v7}2g(M`!pYlfjx?VO5>6|Z^ZYz21 z?&W+vAe#n|NO9KMDe+uE)TrZ%aL&5CWSg&TBFB{S@P_VKJSQWD?FJ?=r_piDwr)u% zQy3`j5UDy^G`dy4<+?$FC!a`?(hKzpky^Gr?G=?P(Dh971-MIIxaZj5>Bj1tbUqL4 zK#BxqlG?a=Zn-zzLzq1i)>Yc74vCI00R5-QdXON1#kNRB0!jbVTk=eKr33aRgREa!h0M(U(d~EopkO>Q9%~eWHU;OyZ%ut)|M!BF%l!wmGgv3T`fiGzZ>h+ls z{ntm4;C9`1t1QZ10_u#DPcWJy3o+XSKo4B@X1}AS=fELDyrMyL!RZ^XzxYLHBz9=P zrQqTcD;TEZ&|zn=cXWK<4ZRx^O^&8{H2|aIg?!Hku8W_)ETz0$tDQ83=w%1$eRw=W z-%|F!mzZbt#ZN|uX2?j*L?4)`4gePrEB5%_IJi`~Blk4i)Gk!QZqVt$eEhK9+US{f zTo=>kMS|I{Tjrct(=WD5LolyCm>pG_1DtbD%th6|b8~;=wEo78{d-HNX#1H=m@CteU