diff --git a/.github/workflows/easy-forest-macos.yml b/.github/workflows/easy-forest-macos.yml new file mode 100644 index 00000000..e2b1b3b0 --- /dev/null +++ b/.github/workflows/easy-forest-macos.yml @@ -0,0 +1,137 @@ +# Easy Forest HDMapping release build for macOS (Apple silicon, macOS 13.0 and newer). +# Separate from the upstream macos.yml: builds only easy_gui + lidar_odometry_step_1 (HDMAPPING_EASY_GUI_ONLY) and +# packages them as "Easy Forest HDMapping.app" in a DMG. The packaging steps are scripts in +# apps/easy_gui/packaging/macos, so they can be run and debugged locally the same way. +name: Easy Forest HDMapping macOS + +on: + push: + tags: ["easy-forest-v*"] + workflow_dispatch: + +permissions: + contents: write # attach the DMG to the release on a tag + +env: + MIN_MACOS: "13.0" + # Picked up by every CMake project (PROJ, the FetchContent raylib) and by clang directly. + MACOSX_DEPLOYMENT_TARGET: "13.0" + +jobs: + build: + runs-on: macos-15 + + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + fetch-depth: 0 + + - name: Set version + run: | + if [[ "$GITHUB_REF" == refs/tags/easy-forest-v* ]]; then + echo "VERSION=${GITHUB_REF_NAME#easy-forest-v}" >> "$GITHUB_ENV" + # The app shows EASY_FOREST_VERSION from CMake; it must be the tag's version. + cmake_version=$(sed -n 's/^set(EASY_FOREST_VERSION "\(.*\)")$/\1/p' apps/easy_gui/CMakeLists.txt) + if [[ "$cmake_version" != "${GITHUB_REF_NAME#easy-forest-v}" ]]; then + echo "::error::Tag $GITHUB_REF_NAME does not match EASY_FOREST_VERSION $cmake_version in apps/easy_gui/CMakeLists.txt" + exit 1 + fi + else + echo "VERSION=$(git describe --tags --always)" >> "$GITHUB_ENV" + fi + + - uses: lukka/get-cmake@latest + with: + cmakeVersion: "~4.0.0" + + # Homebrew's PROJ is built for the runner's macOS; build a static one for MIN_MACOS instead (see the script). + - name: Cache PROJ + id: proj-cache + uses: actions/cache@v4 + with: + path: ${{ runner.temp }}/proj-install + key: proj-macos-${{ env.MIN_MACOS }}-${{ runner.arch }}-${{ hashFiles('apps/easy_gui/packaging/macos/build_proj.sh') }} + + - name: Build PROJ + if: steps.proj-cache.outputs.cache-hit != 'true' + run: apps/easy_gui/packaging/macos/build_proj.sh "$RUNNER_TEMP/proj-install" "$RUNNER_TEMP/proj-work" + + # TBB_DISABLE_HWLOC_AUTOMATIC_SEARCH: the bundled oneTBB would otherwise pick up a Homebrew hwloc for tbbbind. + - name: Configure + run: | + cmake -B build \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_OSX_DEPLOYMENT_TARGET="$MIN_MACOS" \ + -DHDMAPPING_EASY_GUI_ONLY=ON \ + -DBUILD_TESTING=ON \ + -DFREEGLUT_COCOA=ON \ + -DHD_CPU_OPTIMIZATION=ARM \ + -DTBB_DISABLE_HWLOC_AUTOMATIC_SEARCH=ON \ + -DPROJ_DIR="$RUNNER_TEMP/proj-install/lib/cmake/proj" + + # A static library built for a newer macOS only shows up as a linker warning (the executables still say + # minos 13.0), so treat that warning as an error. + - name: Build + run: | + set -o pipefail + cmake --build build --config Release -j"$(sysctl -n hw.ncpu)" 2>&1 | tee build.log + if grep -i "built for newer macOS version" build.log; then + echo "::error::Something was linked that was built for a newer macOS than $MIN_MACOS" + exit 1 + fi + + - name: Test + working-directory: build + run: ctest -C Release --output-on-failure + + # Also checks minos <= MIN_MACOS for every bundled binary and fails otherwise. + - name: Create app bundle + run: python3 apps/easy_gui/packaging/macos/make_app.py build/bin dist --version "$VERSION" --min-macos "$MIN_MACOS" + + - name: Smoke test the bundle + run: | + set +e + "dist/Easy Forest HDMapping.app/Contents/MacOS/easy_gui" --headless > usage.txt 2>&1 + rc=$? + set -e + cat usage.txt + # Exit code 2 + usage text = dyld loaded every bundled library and main() ran. + test "$rc" -eq 2 && grep -q "usage:" usage.txt + + - name: Create DMG + run: apps/easy_gui/packaging/macos/make_dmg.sh "dist/Easy Forest HDMapping.app" dist/EasyForestHDMapping-mac.dmg + + - name: Sizes + run: | + { + echo "### Easy Forest HDMapping macOS $VERSION" + echo '```' + ls -l dist/EasyForestHDMapping-mac.dmg + du -sh "dist/Easy Forest HDMapping.app" + find "dist/Easy Forest HDMapping.app/Contents" -type f -exec ls -l {} \; | awk '{printf "%10d %s\n", $5, substr($0, index($0, $9))}' + echo '```' + } | tee -a "$GITHUB_STEP_SUMMARY" + + - uses: actions/upload-artifact@v4 + with: + name: EasyForestHDMapping-mac-${{ env.VERSION }} + path: dist/EasyForestHDMapping-mac.dmg + compression-level: 0 # already compressed + retention-days: 90 + + # The macOS and Windows workflows both run on the same tag: whichever gets here first creates the release. + # 0.x versions and versions with a suffix (0.2.0-rc1) are pre-releases. Created with GITHUB_TOKEN, so the + # release does not trigger other workflows (upstream windows.yml runs on release: published). + - name: Attach to GitHub release + if: startsWith(github.ref, 'refs/tags/easy-forest-v') + env: + GH_TOKEN: ${{ github.token }} + run: | + PRERELEASE="" + if [[ "$VERSION" == 0.* || "$VERSION" == *-* ]]; then PRERELEASE="--prerelease"; fi + gh release view "$GITHUB_REF_NAME" > /dev/null 2>&1 \ + || gh release create "$GITHUB_REF_NAME" --verify-tag --title "Easy Forest HDMapping $VERSION" $PRERELEASE \ + --notes "Installation and use: [apps/easy_gui/README.md](https://github.com/${GITHUB_REPOSITORY}/blob/${GITHUB_REF_NAME}/apps/easy_gui/README.md)" \ + || gh release view "$GITHUB_REF_NAME" > /dev/null + gh release upload "$GITHUB_REF_NAME" dist/EasyForestHDMapping-mac.dmg --clobber diff --git a/.github/workflows/easy-forest-windows.yml b/.github/workflows/easy-forest-windows.yml new file mode 100644 index 00000000..a99032b2 --- /dev/null +++ b/.github/workflows/easy-forest-windows.yml @@ -0,0 +1,111 @@ +# Easy Forest HDMapping release build for Windows (x64). +# Separate from the upstream windows.yml: builds only easy_gui + lidar_odometry_step_1 (HDMAPPING_EASY_GUI_ONLY) and +# packages them with every DLL they need, the MSVC runtime included, as one zip (apps/easy_gui/packaging/windows). +name: Easy Forest HDMapping Windows + +on: + push: + tags: ["easy-forest-v*"] + workflow_dispatch: + +permissions: + contents: write # attach the zip to the release on a tag + +jobs: + build: + runs-on: windows-latest + + defaults: + run: + shell: bash + + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + fetch-depth: 0 + + - name: Set version + run: | + if [[ "$GITHUB_REF" == refs/tags/easy-forest-v* ]]; then + echo "VERSION=${GITHUB_REF_NAME#easy-forest-v}" >> "$GITHUB_ENV" + # The app shows EASY_FOREST_VERSION from CMake; it must be the tag's version. + cmake_version=$(sed -n 's/^set(EASY_FOREST_VERSION "\(.*\)")$/\1/p' apps/easy_gui/CMakeLists.txt) + if [[ "$cmake_version" != "${GITHUB_REF_NAME#easy-forest-v}" ]]; then + echo "::error::Tag $GITHUB_REF_NAME does not match EASY_FOREST_VERSION $cmake_version in apps/easy_gui/CMakeLists.txt" + exit 1 + fi + else + echo "VERSION=$(git describe --tags --always)" >> "$GITHUB_ENV" + fi + + # Latest CMake: it has to know the generator of whatever Visual Studio the runner image ships. + - uses: lukka/get-cmake@latest + + # HD_CPU_OPTIMIZATION=GENERIC: AUTO adds /arch:AVX2 when the runner's CPU has it, and the release must also run + # on PCs without AVX2. + - name: Configure + run: | + cmake -B build -A x64 \ + -DHDMAPPING_EASY_GUI_ONLY=ON \ + -DBUILD_TESTING=ON \ + -DHD_CPU_OPTIMIZATION=GENERIC \ + -DTBB_DISABLE_HWLOC_AUTOMATIC_SEARCH=ON + + - name: Build + run: cmake --build build --config Release --parallel + + - name: Test + working-directory: build + run: ctest -C Release --output-on-failure + + - name: Create folder and zip + run: | + python apps/easy_gui/packaging/windows/make_zip.py build/bin/Release \ + "dist/Easy Forest HDMapping" dist/EasyForestHDMapping-windows.zip \ + --search-dir build/3rd_binary/PROJ/install_proj/bin + + - name: Smoke test the folder + run: | + # easy_gui.exe is a windowed program, so check from its own folder that it and lidar_odometry_step_1.exe + # load all their DLLs: run with bad arguments and expect the usage exit code 2. + set +e + "dist/Easy Forest HDMapping/easy_gui.exe" --headless + rc=$? + set -e + echo "easy_gui.exe exit code: $rc" + test "$rc" -eq 2 + + - name: Sizes + run: | + { + echo "### Easy Forest HDMapping Windows $VERSION" + echo '```' + ls -l dist/EasyForestHDMapping-windows.zip + du -sh "dist/Easy Forest HDMapping" + ls -l "dist/Easy Forest HDMapping" + echo '```' + } | tee -a "$GITHUB_STEP_SUMMARY" + + - uses: actions/upload-artifact@v4 + with: + name: EasyForestHDMapping-windows-${{ env.VERSION }} + path: dist/EasyForestHDMapping-windows.zip + compression-level: 0 # already compressed + retention-days: 90 + + # The macOS and Windows workflows both run on the same tag: whichever gets here first creates the release. + # 0.x versions and versions with a suffix (0.2.0-rc1) are pre-releases. Created with GITHUB_TOKEN, so the + # release does not trigger other workflows (upstream windows.yml runs on release: published). + - name: Attach to GitHub release + if: startsWith(github.ref, 'refs/tags/easy-forest-v') + env: + GH_TOKEN: ${{ github.token }} + run: | + PRERELEASE="" + if [[ "$VERSION" == 0.* || "$VERSION" == *-* ]]; then PRERELEASE="--prerelease"; fi + gh release view "$GITHUB_REF_NAME" > /dev/null 2>&1 \ + || gh release create "$GITHUB_REF_NAME" --verify-tag --title "Easy Forest HDMapping $VERSION" $PRERELEASE \ + --notes "Installation and use: [apps/easy_gui/README.md](https://github.com/${GITHUB_REPOSITORY}/blob/${GITHUB_REF_NAME}/apps/easy_gui/README.md)" \ + || gh release view "$GITHUB_REF_NAME" > /dev/null + gh release upload "$GITHUB_REF_NAME" dist/EasyForestHDMapping-windows.zip --clobber diff --git a/CMakeLists.txt b/CMakeLists.txt index 8c0ce6b1..16c369c1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,5 +1,11 @@ cmake_minimum_required(VERSION 4.0.0) +# Easy Forest HDMapping release builds (HDMAPPING_EASY_GUI_ONLY) must run on older macOS versions than the +# build machine's, which is the default minimum. Must be set before project(). Pass -DCMAKE_OSX_DEPLOYMENT_TARGET to override. +if(HDMAPPING_EASY_GUI_ONLY AND CMAKE_HOST_APPLE AND NOT CMAKE_OSX_DEPLOYMENT_TARGET) + set(CMAKE_OSX_DEPLOYMENT_TARGET "13.0" CACHE STRING "Minimum macOS version" FORCE) +endif() + project(hd-mapping VERSION 0.105.0) # Common settings @@ -32,6 +38,7 @@ option(BUILD_WITH_BUNDLED_LIBLASZIP "Build with bundled Lib LASZIP" ON) option(BUILD_WITH_BUNDLED_ONETBB "Build with bundled oneTBB" ON) option(BUILD_WITH_PYBIND "Enable pybind11 bindings" OFF) option(HDMAPPING_ASSERT_ALWAYS_ABORT "Make HDMAPPING_ASSERT() abort on failure even in release (NDEBUG) builds" OFF) +option(HDMAPPING_EASY_GUI_ONLY "Build only easy_gui, lidar_odometry_step_1 and their dependencies (plus easy_gui_tests with BUILD_TESTING)" OFF) add_definitions(-D_USE_MATH_DEFINES) @@ -107,37 +114,65 @@ include(cmake/dependencies.cmake) # unconditionally: apps/multi_view_tls_registration (step2) is raylib-based. include(cmake/raylib.cmake) +# HDMAPPING_EASY_GUI_ONLY: the library directories are still added (easy_gui and +# step 1 link some of their targets), but EXCLUDE_FROM_ALL, so a target in them +# is only built when easy_gui or lidar_odometry_step_1 needs it. All other apps, +# tests and optional tool sets are skipped. Off by default: nothing changes. +if(HDMAPPING_EASY_GUI_ONLY) + set(HDMAPPING_EASY_GUI_ONLY_EXCLUDE EXCLUDE_FROM_ALL) + foreach(_hd_skipped BUILD_WITH_CLI_TOOLS BUILD_WITH_HD_MAPPER_APPLICATION BUILD_WITH_UTILITY_APPLICATION) + if(${_hd_skipped}) + message(STATUS "HDMAPPING_EASY_GUI_ONLY: ignoring ${_hd_skipped}") + endif() + endforeach() +endif() + # ============================================================================ # Core Library # ============================================================================ -add_subdirectory(core) -add_subdirectory(calib_core) -add_subdirectory(raylib_widgets) +add_subdirectory(core ${HDMAPPING_EASY_GUI_ONLY_EXCLUDE}) +add_subdirectory(calib_core ${HDMAPPING_EASY_GUI_ONLY_EXCLUDE}) +add_subdirectory(raylib_widgets ${HDMAPPING_EASY_GUI_ONLY_EXCLUDE}) # ============================================================================ # Unit tests (opt-in: cmake -DBUILD_TESTING=ON) # ============================================================================ option(BUILD_TESTING "Build HDMapping unit tests" OFF) -if(BUILD_TESTING) +if(BUILD_TESTING AND HDMAPPING_EASY_GUI_ONLY) + enable_testing() + add_subdirectory(apps/easy_gui/tests) +elseif(BUILD_TESTING) enable_testing() add_subdirectory(shared/tests) add_subdirectory(calib_core/tests) add_subdirectory(apps/lidar_odometry_step_1/tests) add_subdirectory(rosbags/tests) + add_subdirectory(apps/easy_gui/tests) endif() set(CORE_LIBRARIES core) set(GUI_LIBRARIES imgui imguizmo implot) -if(BUILD_WITH_CLI_TOOLS) +if(HDMAPPING_EASY_GUI_ONLY) + # The step 1 directory also defines the drag-and-drop precision forestry app. + # EXCLUDE_FROM_ALL skips it, together with the directory's install rules, so + # lidar_odometry_step_1 is put back into "all" and installed from here. + add_subdirectory(apps/lidar_odometry_step_1 EXCLUDE_FROM_ALL) + set_target_properties(lidar_odometry_step_1 PROPERTIES EXCLUDE_FROM_ALL OFF) + hdmapping_install_app(lidar_odometry_step_1) + add_subdirectory(apps/easy_gui) +endif() + +if(BUILD_WITH_CLI_TOOLS AND NOT HDMAPPING_EASY_GUI_ONLY) add_subdirectory(apps/console_tools) endif() -if(BUILD_WITH_HD_MAPPER_APPLICATION) +if(BUILD_WITH_HD_MAPPER_APPLICATION AND NOT HDMAPPING_EASY_GUI_ONLY) add_subdirectory(apps/hd_mapper) endif() # Applications for typical workflow - see images/workflow.png +if(NOT HDMAPPING_EASY_GUI_ONLY) add_subdirectory(apps/mandeye_raw_data_viewer) add_subdirectory(apps/lidar_odometry_step_1) add_subdirectory(apps/mandeye_single_session_viewer) @@ -147,8 +182,10 @@ add_subdirectory(apps/multi_session_registration) add_subdirectory(apps/multi_session_registration_legacy) add_subdirectory(apps/session_uncertainty_calculation) add_subdirectory(apps/session_to_session_rigid_registration) +add_subdirectory(apps/easy_gui) +endif() -if(BUILD_WITH_UTILITY_APPLICATION) +if(BUILD_WITH_UTILITY_APPLICATION AND NOT HDMAPPING_EASY_GUI_ONLY) add_subdirectory(apps/split_multi_livox) add_subdirectory(apps/precision_forestry_tools) add_subdirectory(apps/compare_trajectories) diff --git a/README.md b/README.md index b8e7817f..54542230 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,7 @@ +> **New: Easy Forest HDMapping.** One window that turns a raw Mandeye recording folder into a point cloud (LAZ) and a +> trajectory (CSV) with the precision forestry settings, no other steps needed. See +> [apps/easy_gui/README.md](apps/easy_gui/README.md). + # Open-source (Windows, Linux and macOS), open hardware hand-held/wearable/mountable mobile mapping system for large-scale surveys: This is open-source software for large-scale 3D mapping using an open-hardware hand-held/wearable/mountable measurement device available at https://github.com/JanuszBedkowski/mandeye_controller. We provide an end-to-end mobile mapping framework that does not require any installation, including: diff --git a/apps/easy_gui/CMakeLists.txt b/apps/easy_gui/CMakeLists.txt new file mode 100644 index 00000000..885e33c1 --- /dev/null +++ b/apps/easy_gui/CMakeLists.txt @@ -0,0 +1,120 @@ +cmake_minimum_required(VERSION 4.0.0) + +project(easy_gui) + +# Easy Forest HDMapping's own version: shown next to HDMapping's in the window title, README.txt, easy_gui.log and +# the LAZ header. Keep it equal to the easy-forest-v* release tag. +set(EASY_FOREST_VERSION "0.2.0") + +# easy_gui_core -- everything except the window: pre-flight check, staging, the +# step 1 child process and its progress parsing, and the pipeline that ties +# them together. A separate static library so the doctest suite in tests/ can +# link it without raylib. +# +# Lidar odometry itself is NOT compiled in: it runs as a child process +# (lidar_odometry_step_1 ), for crash isolation +# and a real cancel -- see src/child_process.h. Only two step 1 sources are +# compiled here, both by relative path like the other apps do: +# - lidar_odometry_utils.cpp for MLvxCalib (the pre-flight check mirrors +# load_data's calibration checks with step 1's own parser), +# - toml_io.cpp to write the parameter file with step 1's own TomlIO. +# toml_io.cpp also reads the parameters back from the session for the export. +# The export (src/export.cpp) loads the step 1 session with core's Session in +# load_cache_mode (poses + trajectories exactly as step 2 sees them, no points) +# and streams the points itself, so core_no_gui is linked below. +# WITH_GUI=0 is PUBLIC because LidarOdometryParams changes layout under +# WITH_GUI (clear_color), so every TU that sees it must agree. +add_library(easy_gui_core STATIC + src/batch.cpp src/batch.h + src/child_process.cpp src/child_process.h + src/export.cpp src/export.h + src/forestry_preset.cpp src/forestry_preset.h + src/las14_writer.cpp src/las14_writer.h + src/pipeline.cpp src/pipeline.h + src/preflight.cpp src/preflight.h + src/progress_parser.cpp src/progress_parser.h + src/settings.h + src/sleep_blocker.cpp src/sleep_blocker.h + src/staging.cpp src/staging.h + src/version.h + ../lidar_odometry_step_1/lidar_odometry_utils.cpp + ../lidar_odometry_step_1/toml_io.cpp +) + +target_compile_definitions(easy_gui_core + PUBLIC WITH_GUI=0 EASY_FOREST_VERSION="${EASY_FOREST_VERSION}" + PRIVATE HDMAPPING_BINARY_PREFIX="${HDMAPPING_BINARY_PREFIX}" UTL_PROFILER_DISABLE) + +target_include_directories(easy_gui_core + PUBLIC + src + ${REPOSITORY_DIRECTORY}/apps/lidar_odometry_step_1 + ${REPOSITORY_DIRECTORY}/core/include + ${THIRDPARTY_DIRECTORY} + ${EIGEN3_INCLUDE_DIR} + ${THIRDPARTY_DIRECTORY}/json/include + ${THIRDPARTY_DIRECTORY}/tomlplusplus/include + ${LASZIP_INCLUDE_DIR}/LASzip/include + ${THIRDPARTY_DIRECTORY}/observation_equations/codes + ${THIRDPARTY_DIRECTORY}/vqf/vqf/cpp + ${THIRDPARTY_DIRECTORY}/Fusion/Fusion) + +# core_math provides the Core/ndt.h + hash_utils symbols lidar_odometry_utils.cpp +# needs (the same reason step 1's own tests link core_no_gui); TBB and laszip for +# its load_point_cloud()/decimate(), which the pre-flight check never calls. +target_link_libraries(easy_gui_core PUBLIC + core_no_gui + core_math + vqf + Fusion + unordered_dense::unordered_dense + spdlog::spdlog + UTL::include + ${PLATFORM_LASZIP_LIB} + ${PLATFORM_MISCELLANEOUS_LIBS}) + +# Batch mode keeps the computer awake (src/sleep_blocker.cpp): IOKit power assertions on macOS. +if(APPLE) + target_link_libraries(easy_gui_core PUBLIC "-framework IOKit" "-framework CoreFoundation") +endif() + +if(MSVC) + target_compile_definitions(easy_gui_core PUBLIC _USE_MATH_DEFINES NOMINMAX) + target_compile_options(easy_gui_core PRIVATE /bigobj) +endif() + +# The window. raylib + rlImGui like step 2; core_pfd for the native folder picker. +# imgui_stdlib (std::string InputText, for the batch table's output names) is compiled here: imgui_raylib does not +# include it, and it must come from the same vendored ImGui. +add_executable(easy_gui src/easy_gui_main.cpp ${THIRDPARTY_DIRECTORY}/imgui/misc/cpp/imgui_stdlib.cpp) + +target_link_libraries(easy_gui PRIVATE + easy_gui_core + core_pfd + raylib_widgets + raylib + imgui_raylib + rlimgui) + +# The step 1 program is part of Easy GUI at run time (it is started as a child +# process from the same directory), so building Easy GUI builds it too. +add_dependencies(easy_gui lidar_odometry_step_1) + +if(WIN32) + # A windowed program: no console window behind the GUI. main() stays the entry point (the WIN32 subsystem would + # otherwise expect WinMain); --headless/--export-only print into the console they were started from + # (attachToParentConsole). The manifest makes UTF-8 the process code page, see src/easy_gui.manifest. + set_target_properties(easy_gui PROPERTIES WIN32_EXECUTABLE ON) + if(MSVC) + target_link_options(easy_gui PRIVATE /ENTRY:mainCRTStartup) + endif() + target_sources(easy_gui PRIVATE src/easy_gui.manifest) + + add_custom_command( + TARGET easy_gui + POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy $ $ + COMMAND_EXPAND_LISTS) +endif() + +hdmapping_install_app(easy_gui) diff --git a/apps/easy_gui/README.md b/apps/easy_gui/README.md new file mode 100644 index 00000000..0635d242 --- /dev/null +++ b/apps/easy_gui/README.md @@ -0,0 +1,329 @@ +# Easy Forest HDMapping + +Easy Forest HDMapping is a single window that turns the folder a Mandeye scanner records into a 3D point cloud (a `.laz` file) +and the path you walked (a `.csv` file). It uses the processing settings that HDMapping recommends for forests, so there +is nothing to tune. + +![The Easy Forest HDMapping window after choosing a recording](docs/images/easy_gui_window.png) + +- [Who it is for](#who-it-is-for) +- [What your computer needs](#what-your-computer-needs) +- [Installation](#installation) +- [How to use it](#how-to-use-it) +- [Processing many recordings](#processing-many-recordings) +- [What you get](#what-you-get) +- [Troubleshooting](#troubleshooting) +- [Getting help](#getting-help) +- [Credits and license](#credits-and-license) + +## Who it is for + +Easy Forest HDMapping is for anyone who walks a forest plot with a Mandeye scanner and wants a point cloud without learning the +full HDMapping toolchain: forestry partners, students, field crews. It is designed for forests; it works elsewhere, but +its settings are tuned for forests. + +What it does **not** do: + +- **One point cloud per walk.** Each recording becomes its own point cloud. It can process many recordings in one go + (see [Processing many recordings](#processing-many-recordings)), but it does not merge several walks into one cloud. +- **No real-world coordinates.** It does not use GNSS or ground control points. Coordinates are in metres, starting at + the spot where you started walking, with the vertical axis (Z) pointing up. +- **No loop closure yet.** If you return to your starting point, small drift is not corrected. (Planned for a later + version.) +- **One lidar only.** Scanners with two or more lidars are refused by the check. + +For any of these, use the full HDMapping apps (see the +[HDMapping wiki](https://github.com/MapsHD/HDMapping/wiki)). Easy Forest HDMapping keeps its intermediate results so that you +can continue there. + +## What your computer needs + +| | Mac | Windows | +|---|---|---| +| System | macOS 13 (Ventura) or newer | Windows 10 or 11, 64-bit (not yet tested on a real PC) | +| Processor | Apple silicon (M1 or newer). Intel Macs are not supported in v1. | 64-bit Intel or AMD processor | +| Memory (RAM) | 16 GB recommended | 16 GB recommended | + +**Memory:** a 10-minute walk needs about 9 GB of memory while it is being processed. Longer walks need more. Close +other large programs while Easy Forest HDMapping is working. + +**Disk space, rule of thumb:** while processing, Easy Forest HDMapping needs free space of about **20 times the size of the +recording's lidar files**, which is roughly **1.5 GB per minute of walking**. About a tenth of that stays as results; +the rest is freed when processing ends. You do not have to work this out yourself: the check (step 3 below) tells you +exactly how much is needed and how much is free. + +**Time:** on a recent laptop, processing takes about twice as long as the walk (a 10-minute walk takes about +20-25 minutes). + +## Installation + +Easy Forest HDMapping is free and needs no account. Your computer will warn you the first time you open it, because the program +is not signed by a company that Apple or Microsoft knows. The steps below show how to get past the warning; you only +need to do this once. + +### Mac + +1. Open the [releases page](https://github.com/MartinMokros/HDMapping-Easy-GUI/releases) and download the newest Easy Forest HDMapping release: the file + **`EasyForestHDMapping-mac.dmg`** under *Assets*. +2. Double-click the `.dmg` file. In the window that opens, drag **Easy Forest HDMapping** onto the **Applications** + folder. +3. Open your **Applications** folder and double-click **Easy Forest HDMapping**. +4. macOS says **"“Easy Forest HDMapping” Not Opened"** (Apple could not verify it is free of malware). Click + **Done**. +5. Open **System Settings** → **Privacy & Security**. Scroll down to the **Security** section. Next to the line + **"“Easy Forest HDMapping” was blocked to protect your Mac."** click **Open Anyway**. +6. Enter your Mac password (or use Touch ID), then click **Open Anyway** once more in the dialog that appears. + +From now on Easy Forest HDMapping opens normally. If macOS ever says the app **"is damaged and can't be opened"**, see +[Troubleshooting](#troubleshooting). + +### Windows (not yet tested on a real PC) + +The Windows version is built and its automatic tests pass, but nobody has used it on a real Windows PC yet. If +something goes wrong, please report it (see [Getting help](#getting-help)). + +1. Open the [releases page](https://github.com/MartinMokros/HDMapping-Easy-GUI/releases) and download the newest Easy Forest HDMapping release: the file + **`EasyForestHDMapping-windows.zip`** under *Assets*. +2. Before unpacking: right-click the downloaded `.zip` file → **Properties** → on the **General** tab, tick + **Unblock** at the bottom → **OK**. (If there is no Unblock box, skip this step.) +3. Right-click the `.zip` file → **Extract All…** → choose a folder, for example `Documents\Easy Forest HDMapping` + → **Extract**. Keep all files in that folder together: Easy Forest HDMapping needs the other files next to it. +4. Open the folder and double-click **`easy_gui.exe`**. +5. If a blue window says **"Windows protected your PC"**: click **More info**, then **Run anyway**. + +Tip: right-click `easy_gui.exe` → **Show more options** → **Send to** → **Desktop (create shortcut)** to start it +from your desktop next time. + +## How to use it + +At the top of the window, choose **One recording** (described here) or **Several recordings** (see +[Processing many recordings](#processing-many-recordings)). The window guides you through four steps, top to bottom. + +**1. Recording.** Click **Choose recording…** and select the folder the scanner wrote, for example +`continousScanning_0012`. It contains files named `lidar0000.laz`, `imu0000.csv` and so on. You can also drag the +folder onto the window. You can work straight from the scanner's USB stick: Easy Forest HDMapping never changes or adds +anything in the recording folder. Copying the folder to your computer first makes processing a little faster. + +**2. Output folder.** Easy Forest HDMapping suggests `Documents/Easy Forest HDMapping/`. To use a +different place, click **Choose output folder…**. The folder must be new or empty. + +**3. Check.** Easy Forest HDMapping checks the recording and the free disk space automatically. Each line is marked: + +- **[OK]** (green): all good. +- **[!]** (amber): worth knowing, but you can go on. +- **[X]** (red): this must be fixed before you can start. See [Troubleshooting](#troubleshooting). + +**4. Process.** Click **Start processing**. Easy Forest HDMapping shows which stage it is in, a progress bar and the time left. +You can keep using your computer, but it will be busy. **Cancel** stops processing and deletes the unfinished results. +Closing the window also stops processing. + +When it is done, click **Open output folder** to see the results, or **Process another recording** to start again. + +**Advanced settings** (you normally do not need these): + +- **Closest points kept** (default 0.4 m): points closer to the scanner than this are left out. The default removes + the operator's body and the scanner handle. A lower value keeps more of the close surroundings, but may include you. +- **Place all points by their measurement time (experimental)**: an alternative way of placing the points. Tests showed + it slightly less sharp than the default, so leave it off unless you are comparing the two. +- **Keep intermediate files**: keeps extra working files, useful when reporting a problem. + +## Processing many recordings + +After a field day you often have many recordings, for example one per plot. Easy Forest HDMapping can process all of +them one after another, unattended, for example overnight. + +![Several recordings: three recordings found, all ready to process](docs/images/easy_gui_batch.png) + +1. At the top of the window, choose **Several recordings**. +2. **Folder with recordings.** Click **Choose folder…** and select the folder that *contains* the recording folders, + for example a copy of the scanner's USB stick. You can also drag it onto the window. Easy Forest HDMapping looks in + the folders directly inside it (not deeper). If the folder you chose is itself a recording, it is included too. +3. **Output folder.** Easy Forest HDMapping suggests `Documents/Easy Forest HDMapping/`. Each + recording's results go into a folder of its own inside it. +4. **Recordings.** A table lists every recording found: + + | Column | Meaning | + |---|---| + | Include | Untick a recording to leave it out. Recordings that fail the check cannot be ticked. | + | Recording | The folder name. Point at it to see the full path. | + | Length | How long the walk was. | + | Check | **[OK]**, **[!]** (worth knowing, you can go on) or **[X]** (cannot be processed). Point at it for the details; problems are also listed in words below the table. **already done** means this recording was finished in an earlier run and will be skipped. | + | Output folder name | The name of this recording's results folder. You can change it. Two recordings cannot use the same name (upper and lower case count as the same). | + + Folders that are not recordings are listed under **folders skipped**, with the reason. Below the table, the disk + check tells you how much space the whole batch needs: the temporary space of the largest recording (recordings are + processed one at a time, and each frees its temporary space when it is done) plus the results of all of them. +5. **Process.** Click **Start processing**. You see the progress of the whole batch with the time left, which + recording is being processed, and the status of each recording in the table. Your computer will not go to sleep + while the batch runs (the screen may still turn off). Leave the power adapter connected. + + ![A batch in progress: two recordings done, the third running](docs/images/easy_gui_batch_running.png) + +If a recording fails, it is marked **failed** and the batch goes on with the next one. Point at **failed** in the table +to see why; the recording's own `easy_gui.log` has the details. + +**Stopping.** **Stop after this recording** lets the current recording finish and then stops. **Stop now** stops +immediately and discards the current recording; recordings already finished are kept. + +**Continuing a stopped batch.** Choose the same folder with recordings and the same output folder again. Recordings +that were finished show **already done** and are skipped; the others are processed, including the ones that failed +before. This also works after a power cut or a crash. (Each finished recording's folder contains a small file, +`easy_gui_done.txt`, that marks it as finished. Delete that file to process the recording again.) + +**What you get.** The output folder contains one folder per recording, each with the files described in +[What you get](#what-you-get), plus two files about the whole batch: + +| File | What it is | +|---|---| +| `batch_summary.csv` | One row per recording: recording folder, output folder name, status (`succeeded`, `failed`, `already_done`, `cancelled`, `not_run`, `excluded`, `skipped`), processing time in seconds, number of raw points, number of points in the point cloud, results folder, and the error message if any. Opens in any spreadsheet program. | +| `batch.log` | What happened in the batch, with times. | + +**Without the window.** The batch can also be started from a terminal, for example on a computer without a screen: + +``` +easy_gui --batch +``` + +On a Mac the program is `"/Applications/Easy Forest HDMapping.app/Contents/MacOS/easy_gui"`; on Windows, +`easy_gui.exe` in the folder you extracted. It follows the same rules as the window: every recording that passes the +check is processed, and finished ones are skipped. Press Ctrl+C once to stop after the current recording, twice to stop +now. The program ends with exit code 0 if every recording succeeded or was already done, and 1 otherwise. + +## What you get + +The output folder contains: + +| File | What it is | +|---|---| +| `point_cloud.laz` | The point cloud. Opens in CloudCompare, QGIS, LAStools, lidR and most other point cloud software. | +| `trajectory.csv` | The path of the scanner: where it was and which way it pointed, about 200 times a second. Opens in any spreadsheet program. | +| `README.txt` | A description of both files, with the numbers of this particular run. | +| `easy_gui.log` | A record of everything that happened during processing. Useful when asking for help. | +| `work/` | Intermediate results. `work/lio_session/session.mjs` can be opened in the full HDMapping apps to improve the result further or to give it real-world coordinates. | + +### The point cloud (`point_cloud.laz`) + +Every point has these values: + +| Field | Meaning | +|---|---| +| X, Y, Z | Position in metres. (0, 0, 0) is where you started walking; Z points up. | +| intensity | How strongly the surface reflected the laser. Bright, smooth surfaces give high values. | +| gps_time | When the point was measured, in seconds since 1 January 1970 (UTC). Despite the name, this is not GPS time. | +| point_source_id | Which roughly 5-second piece of the walk the point belongs to. | +| user_data | Which lidar measured the point (always the same, since Easy Forest HDMapping handles one lidar). | +| **range_m** | **Distance from the scanner to the point when it was measured, in metres.** Far points are less precise than near ones, so this is useful for keeping only points measured from close by, for example only points closer than 15 m for measuring stem diameters. | +| **point_origin** | **Where the point comes from:** | +| | **0** — the main points, placed by HDMapping's motion tracking (lidar odometry). Most of the cloud. | +| | **1** — close-range points. HDMapping's motion tracking ignores everything within 1 m of the scanner; Easy Forest HDMapping adds these points back afterwards (from the "Closest points kept" distance outwards), so the ground and stems right next to your path are not missing. | +| | **2** — filled-in points. HDMapping's motion tracking works in tenth-of-a-second pieces and skips a few milliseconds between them; Easy Forest HDMapping adds the points measured in those short gaps. | + +Points 1 and 2 line up with the main points typically within a millimetre, and in 99 % of cases within about 1 cm. +You can switch them off by filtering on `point_origin`, for example to see exactly what HDMapping alone would have +produced. + +### The trajectory (`trajectory.csv`) + +One row per scanner position, with a header row. The most useful columns: + +| Column | Meaning | +|---|---| +| timestamp_unix_s | Time, in the same seconds as the point cloud's gps_time. | +| x, y, z | Position of the scanner in metres, in the same coordinates as the point cloud. | +| qw, qx, qy, qz | Which way the scanner pointed, as a quaternion. | +| om_deg, fi_deg, ka_deg | The same direction as three angles in degrees. | +| scan_index | Matches point_source_id in the point cloud. | + +`README.txt` describes the remaining columns. + +### Colouring the point cloud by a field in CloudCompare + +[CloudCompare](https://www.cloudcompare.org) is free and opens `point_cloud.laz` directly. + +1. **File → Open**, choose `point_cloud.laz`. In the dialog that lists the fields, keep everything ticked and click + **Apply all**. +2. Click the point cloud in the **DB Tree** panel on the left. +3. In the **Properties** panel below it, find **Colors** and choose **Scalar field**. +4. Just below, under **Scalar Fields**, set **Active** to `range_m` or `point_origin` (or `Intensity`). +5. To keep only some points, for example only the main points: **Edit → Scalar fields → Filter By Value**, enter a + range (for example 0 to 0 for `point_origin`), and click **Export**. CloudCompare creates a new cloud with only + those points. + +## Troubleshooting + +The check (step 3) explains problems in plain words. Here is what to do about each one. + +**Problems with the recording** + +| Message (shortened) | What to do | +|---|---| +| "The recording folder … does not exist or is not a folder." | Choose the folder again; the USB stick may have been unplugged. | +| "This folder does not look like a Mandeye recording: there are no point cloud files / no IMU files." | You chose the wrong folder. Choose the folder that contains `lidar….laz` and `imu….csv` files, for example `continousScanning_0012`, not the folder above or below it. | +| "There are N point cloud files but M IMU files. The unmatched file(s) at the end will not be processed." | Just information. Usually the scanner stopped between two files; the last few seconds are left out. | +| "Some file numbers do not match." | A file is probably missing in the middle of the recording (for example after an incomplete copy). Copy the recording from the scanner again. If that does not help, the result may have a gap at that point. | +| "These point cloud files cannot be read (damaged, or not fully copied from the scanner)." | Copy the recording again. If the damaged files are the last ones, you can also delete them from your copy. | +| "This recording contains data from more than one lidar." / "…describes more than one lidar." / "…lists more than one lidar." | Easy Forest HDMapping handles single-lidar scanners only. Use the full HDMapping apps. | +| "The calibration file … could not be read and will be ignored." | Just information, processing continues without it. If your scanner needs a calibration file, ask whoever set up the scanner for a correct one. | +| "The calibration file … belongs to a different scanner." / "… does not name a valid IMU." | The calibration file in the folder does not match the scanner. Remove it from your copy of the recording, or replace it with the right one. | +| "The IMU files could not be read." | The motion-sensor files (`imu….csv`) are damaged or empty. Copy the recording again. | +| "The IMU and point cloud timestamps do not overlap." | The recording is damaged; the two kinds of files do not belong together. Copy it again from the scanner. | +| "The recording is very short." | Just a warning. Processing needs some walking; recordings of less than about 20 seconds may give poor results. | + +**Problems with the output folder** + +| Message (shortened) | What to do | +|---|---| +| "Choose an output folder." | Click **Choose output folder…**. | +| "The output folder is inside the recording folder." | Choose a folder somewhere else. Easy Forest HDMapping never writes into the recording. | +| "The output folder … already contains files." | Choose a new or empty folder, or empty this one first. | +| "Not enough free disk space …" | Free some space (the message says how much is needed), or choose an output folder on another disk with more space. | +| "Free disk space is tight." | You can start, but make sure nothing else fills the disk while Easy Forest HDMapping works. | +| "Could not determine the free disk space." | Usually harmless. Make sure the disk has plenty of space (see the rule of thumb above). | +| "The output folder holds an unfinished earlier attempt." (several recordings) | Just information: the recording did not finish last time. Its old files are replaced; its log is continued. | +| "Not enough free disk space for the batch …" (several recordings) | Free some space, choose an output folder on another disk, or untick some recordings and process them later. | +| "The output folder is inside the recording …" (several recordings) | The folder with recordings is itself a recording, and the output folder is inside it. Choose an output folder somewhere else. | +| Output folder name shown in red (several recordings) | The name is empty, contains a character a folder name cannot have (such as `/`, `\`, `:`, `*`, `?` or a vertical bar), or another recording uses it. Point at it to see which. | + +**Problems during processing** + +| Message | What to do | +|---|---| +| "Easy Forest HDMapping could not find the lidar odometry program …" | Some files of the installation are missing. Install Easy Forest HDMapping again and keep all its files together. | +| "Lidar odometry stopped unexpectedly." | Most often the computer ran out of memory. Close other programs and try again. If it happens again, see [Getting help](#getting-help). | +| "Could not create the output folder …" / "Could not write …" | The disk may be full, or you may not be allowed to write there. Choose another output folder. | + +**Mac: "“Easy Forest HDMapping” is damaged and can't be opened."** This sometimes happens with downloaded apps that +are not signed. Open the **Terminal** app, paste the following line and press Enter, then open Easy Forest HDMapping again: + +``` +xattr -dr com.apple.quarantine "/Applications/Easy Forest HDMapping.app" +``` + +## Getting help + +Open an issue at and attach `easy_gui.log` from the +output folder. Describe what you did and which message you saw. + +## Credits and license + +Easy Forest HDMapping is built on [HDMapping](https://github.com/MapsHD/HDMapping) by Janusz Będkowski and the MapsHD +contributors. All the processing (motion tracking and point placement) is HDMapping's; Easy Forest HDMapping adds the window, +the checks and the export. Easy Forest HDMapping and HDMapping are released under the [MIT License](../../LICENSE). + +If you use the results in your work, please cite HDMapping: + +```bibtex +@article{BEDKOWSKI2024101618, + title = {Open source, open hardware hand-held mobile mapping system for large scale surveys}, + journal = {SoftwareX}, + volume = {25}, + pages = {101618}, + year = {2024}, + issn = {2352-7110}, + doi = {https://doi.org/10.1016/j.softx.2023.101618}, + url = {https://www.sciencedirect.com/science/article/pii/S235271102300314X}, + author = {Janusz Będkowski} +} +``` + +How Easy Forest HDMapping works and how its results were checked is described in [docs/DESIGN.md](docs/DESIGN.md). diff --git a/apps/easy_gui/docs/DESIGN.md b/apps/easy_gui/docs/DESIGN.md new file mode 100644 index 00000000..17bc1ac1 --- /dev/null +++ b/apps/easy_gui/docs/DESIGN.md @@ -0,0 +1,552 @@ +# Easy GUI: design notes + +This document explains how Easy GUI works, why it is built the way it is, and how its output was validated. It is for +collaborators and reviewers who know HDMapping. The user guide is [../README.md](../README.md). + +Naming: the user-facing name is **Easy Forest HDMapping** (window title, user guide, release packages). Internally, +the code keeps the name Easy GUI (`apps/easy_gui`, the `easy_gui` binary and targets), and so does this document. + +Version: Easy Forest HDMapping has its own version (`EASY_FOREST_VERSION` in `apps/easy_gui/CMakeLists.txt`, +equal to the `easy-forest-v*` tag). Window title, README.txt and easy_gui.log show both, e.g. "Easy Forest HDMapping +0.2.0 (HDMapping 0.105.0)". The LAZ header's 32-character fields cannot hold that: generating software is +"Easy Forest HDMapping 0.2.0", system identifier "HDMapping 0.105.0". + +Contents: + +1. [Goal and scope](#1-goal-and-scope) +2. [Architecture](#2-architecture) +3. [Background: Mandeye raw data](#3-background-mandeye-raw-data) +4. [Background: what lidar odometry (step 1) does to points](#4-background-what-lidar-odometry-step-1-does-to-points) +5. [Design decisions](#5-design-decisions) +6. [The export](#6-the-export) +7. [Validation](#7-validation) +8. [Experimental option: place every point by its raw time](#8-experimental-option-place-every-point-by-its-raw-time) +9. [Resources: time, memory, disk](#9-resources-time-memory-disk) +10. [Batch mode](#10-batch-mode) +11. [Status and next steps](#11-status-and-next-steps) + +--- + +## 1. Goal and scope + +One window that takes a raw Mandeye recording folder, runs HDMapping's lidar odometry (step 1) with the precision +forestry preset, and exports two files: + +- a LAZ point cloud with every per-point attribute that is available, and +- a trajectory CSV with timestamp, position, orientation and every other available field. + +Constraints: + +- Built on the raylib + rlImGui GUI stack (like steps 2 and 3). +- Reuse HDMapping's code; do not duplicate its algorithms. `core_math` is not modified, and the existing apps behave + as before. +- Show progress and report errors in plain language. + +Since 0.2.0 it also processes several recordings in one go (batch mode, §10), each into its own output folder. + +Not in scope (v1): step 2 refinement, step 3 (multi-session), georeferencing (GNSS, control points), and rigs with +more than one lidar. The pre-flight check refuses recordings with more than one lidar id. + +Planned for v2: pose-graph loop closure behind a toggle, using `PoseGraphLoopClosure` with automatically detected +candidates, where an edge is rejected if its ICP fails. `PoseGraphSLAM::optimize` will not be used. + +## 2. Architecture + +``` +recording folder (read only) + │ + ├─ pre-flight check ────────── file pairing, LAZ/IMU readability, lidar ids, calibration, clocks, disk space + │ + ├─ staging ─────────────────── output/work/staging/: symlinks to the recording's files + │ + ├─ lidar_odometry_step_1 ───── child process: + │ (forestry preset) progress parsed from its log + │ + └─ export ──────────────────── point_cloud.laz, trajectory.csv, README.txt +``` + +Code in `apps/easy_gui`: + +| Part | Role | +|---|---| +| `easy_gui_core` (static library, no raylib) | Everything except the window, so the doctest suite can link it. | +| `src/preflight.*` | Classifies files exactly like step 1's `load_data` does and checks them (see §3). | +| `src/staging.*` | Builds the staging folder of symlinks (see §5.6). | +| `src/child_process.*` | Runs step 1: POSIX `posix_spawn` with its own process group; Windows `CreateProcess` with a job object (built and tested on the Windows CI runner). | +| `src/progress_parser.*` | Turns step 1's log lines into phases and a progress fraction. | +| `src/forestry_preset.*` | Writes the step 1 parameter file with step 1's own `TomlIO`. | +| `src/export.*`, `src/las14_writer.*` | The export (§6) and a streaming LAS 1.4 point format 6 LAZ writer with a hand-written extra-bytes VLR. | +| `src/pipeline.*` | Worker thread that runs the stages above and publishes status for the GUI. | +| `src/batch.*` | Batch mode (§10): discovery, output names, done markers, batch disk check, and `BatchRunner`, which runs `Pipeline` once per recording. | +| `src/sleep_blocker.*` | Keeps the computer awake while a batch runs (§10.5). | +| `src/easy_gui_main.cpp` | The window, plus the command-line modes below. | +| `tests/` | `easy_gui_tests` (doctest). | +| `scripts/check_export.py` | Acceptance checks for an export (§7). | + +`easy_gui_core` compiles two step 1 sources by path, like the other apps do: `lidar_odometry_utils.cpp` (for the +calibration parser the pre-flight check uses) and `toml_io.cpp` (to write the parameter file and to read it back from +the session). It is built with `WITH_GUI=0` and links `core_no_gui` (for `Session`) and `core_math`, not `core`. +Because of that, `core_no_gui` is now always built; before, it was only built with tests, pybind or the CLI tools. + +**Why step 1 runs as a child process.** Crash isolation (step 1 calls `std::exit` on some input errors and opens a +native message box on `bad_alloc`), and a real cancel: killing the process group stops it immediately and frees its +~8.5 GB. Building `easy_gui` also builds `lidar_odometry_step_1` (`add_dependencies`). Easy GUI finds it next to its own +executable, under the plain name or the `HDMAPPING_BINARY_PREFIX` name; `HDMAPPING_STEP1_EXECUTABLE` overrides the +lookup. + +**Building only Easy GUI:** `cmake -B build -DHDMAPPING_EASY_GUI_ONLY=ON` builds and installs only `easy_gui`, +`lidar_odometry_step_1` and the libraries they link (plus `easy_gui_tests` with `BUILD_TESTING=ON`). The default is +off, which leaves the upstream build unchanged. On macOS this mode also sets `CMAKE_OSX_DEPLOYMENT_TARGET` to 13.0 +(unless given), the lowest version the code builds for; without it the minimum is the build machine's macOS version. + +Quirks of headless step 1 (`lidar_odometry_step_1 `) that Easy GUI handles: + +- The exit code is 0 even after a caught exception, so success is judged by the presence of `session.mjs`. +- `load_data` waits on stdin before `std::exit` when the calibration does not match, so the child gets a closed stdin. + The pre-flight check repeats `load_data`'s calibration checks so that this case is refused before step 1 starts. +- Progress exists only as log lines: `running iterations: i/n` during setup and `finished optimizing … - i / N …` per + odometry chunk. + +**Command-line modes** of `easy_gui`: + +| Mode | Purpose | +|---|---| +| `--headless ` | The whole pipeline without a window. Exit code 0 success, 1 failed or refused, 2 usage. | +| `--batch ` | Batch mode without a window (§10). Exit code 0 when every selected recording succeeded or was already done, 1 otherwise or refused, 2 usage. Ctrl+C once = stop after current, twice = stop now. | +| `--export-only ` | Only the export, on an existing step 1 session (about 2.5 min instead of about 22). | +| `--check-band ` (with any of the above) | Test mode for the alignment check, see §7.3. | +| `--place-by-raw-time` (with any of the above) | The experimental export option of §8. | +| `--screenshot [recording \| --batch ]` | Renders the window for about 2 s, saves it and exits (for documentation). raylib saves into the working directory. `EASY_GUI_SCREENSHOT_START=1` starts the batch once it is ready and `EASY_GUI_SCREENSHOT_FRAME=` takes the picture at frame n (30 per second), to capture a running or finished batch. | + +**Output folder:** + +| Path | Content | +|---|---| +| `point_cloud.laz` | The point cloud (§6). | +| `trajectory.csv` | The trajectory (§6.4). | +| `README.txt` | Field descriptions, time base and near-point rule, generated with the run's own values. | +| `easy_gui.log` | Everything, including step 1's full output. | +| `work/easy_gui_step1.toml` | The parameters step 1 ran with. | +| `work/lio_session/` | The step 1 session (`session.mjs` etc.). Kept, so it can be opened in step 2 for refinement or georeferencing. | +| `work/staging/` | Removed after success unless "Keep intermediate files" is ticked. | +| `easy_gui_done.txt` | Batch mode only: written once the result is complete (§10.4). | + +After a failure or a cancel, `work/` is removed, except that an export failure keeps `work/lio_session/`, since +odometry took most of the run. + +## 3. Background: Mandeye raw data + +Verified on a real recording and in the Mandeye firmware source. + +**LAZ files** (`lidar*.laz`): LAS 1.2, point format 1. + +- Real fields: X, Y, Z in the sensor frame, intensity (0–160 in practice), gps_time, and user_data = lidar id. There + are no extra bytes; every other standard field is 0. +- gps_time is Unix epoch seconds. The header wrongly flags it as adjusted GPS time. +- classification is a constant 20 and has no meaning: the firmware never copies the Livox point tag. A planned firmware + fix will store the tag there (bits 0–1 adhesion, 2–3 particles, 4–5 other). +- About 15 % of the records are "no return" points: exactly (0, 0, 0) or gps_time 0. + +**IMU files** (`imu*.csv`, 200 Hz): timestamp (ns, the same clock as the LAZ gps_time), gyro XYZ, acc XYZ (g), imuId, +and timestampUnix (ms, the host computer's clock, not synchronised with the others). + +**How step 1's `load_data` picks files** (the pre-flight check mirrors this): + +- It sorts all files, then pairs `lidar*.laz` with `imu*.csv` **by position**, and loads `min(#laz, #csv)` pairs. A + mismatch of the 4-digit index only prints a warning; surplus trailing files are skipped. +- `status*` and `cam*` JSON files are ignored. Any other `.json`, or a `.mjc`, is taken as the calibration file. Only + the first `.sn` file is used. +- It writes `cache/` and `preview/` into the folder that holds the input files. + +## 4. Background: what lidar odometry (step 1) does to points + +Verified in `compute_step_1` and `save_result` (`apps/lidar_odometry_step_1`). These facts drive most decisions in §5. + +**Timestamps are not preserved.** Step 1 works in worker chunks of 20 trajectory poses (≈ 0.1 s). In each chunk it +overwrites every point's timestamp with a uniform ramp `t0 + k·(t_last − t0)/N` ("correct points timestamps" in +`lidar_odometry.cpp`). Raw gps_time therefore cannot be joined to step 1's output points, with or without a tolerance. + +Output formats: gps_time in `scan_lio_N.laz` and the timestamps in `trajectory_lio_N.csv` are seconds × 1e9 (ns stored +as a double, ≈ 256 ns resolution). `scan_lio_N.laz` holds only XYZ (in the frame of the scan's first trajectory node), +intensity and gps_time; the lidar id is dropped in `points_to_vector`. + +**Distance filters:** + +- `filter_threshold_xy_inner` / `_outer` act at load time (`load_point_cloud`), so they remove points from the odometry + input *and* from the output. They measure the **horizontal** radius √(x² + y²) in the raw sensor frame. +- `threshould_output_filter` is **output only** (`points_to_vector` in `save_result`, `fill_session_from_worker_data`; + the optimisers never read it) and measures the **3D** norm in the sensor frame. +- Consequently an output filter below `filter_threshold_xy_inner` has no effect: every surviving point already has a 3D + norm ≥ its horizontal radius > inner. + +**Other points step 1 drops:** gps_time 0; points before the first `threshold_initial_points`; the gap +`[t_last, t_next_first)` between consecutive worker chunks (≈ 1 of every 20 IMU intervals, about 5 % of the time); the +first point of each chunk; chunks with ≤ 1000 points; the trailing poses that do not fill a whole chunk; and the last +concatenated group if it has ≤ `min_counter_concatenated_trajectory_nodes` nodes. + +Voxel decimation (`params.decimation`) only thins the points used for optimisation; `scan_lio_N.laz` is written from +the undecimated originals. + +**Per-point range is recoverable without the raw files:** `index_pose` = the last trajectory node with t_node < t_point +(step 1's own rule, applied to the ramp time), then `p_sensor = csv_pose[index_pose]⁻¹ · p_laz`, both relative to the +scan's first node. (`save_index_pose` would record the index directly, but it is not in `TomlIO`'s attribute list, so a +parameter file cannot enable it.) + +## 5. Design decisions + +### 5.1 Output format: LAS 1.4, point format 6, extra bytes + +Point format 6 has a 64-bit gps_time and room for extra bytes. Two extra-bytes attributes are written: + +- `range_m` (float32): 3D distance from the lidar at the moment of measurement. +- `point_origin` (uint8): where the point comes from (0 = step 1, 1 = near point, 2 = gap point, 3 = test mode only). + +The extra-bytes VLR is written by hand because `laszip_add_attribute` sets the scale bit, which makes readers such as +laspy decode the fields as float64. With the hand-written VLR, CloudCompare 2.x loads both as named scalar fields and +laspy reads them as float32 / uint8. + +`user_data` keeps the lidar id; `point_source_id` is the step 1 scan index (the `scan_lio_N.laz` number, about 5 s of +recording). + +### 5.2 Time base + +`gps_time` is Unix epoch seconds, not GPS week time and not adjusted standard GPS time, so global encoding bit 0 (GPS +time type) is 0. README.txt states this. For the default export, step 1's own points carry step 1's ramp time (§4), +typically 1–4 ms and at most about 20 ms from the real measurement time; near and gap points carry their raw time. + +(Step 2's own `--export` writes gps_time × 1e18, because it multiplies the session's ns-valued times by 1e9 again. +Easy GUI does not reproduce that.) + +### 5.3 The default export is "hybrid" + +The point cloud combines: + +- `point_origin = 0`: step 1's own points, placed exactly as step 1 placed them; +- `point_origin = 1`: **near points**, raw points step 1's distance filters removed (§5.4); +- `point_origin = 2`: **gap points**, raw points inside step 1's distance limits that step 1 skipped between its 0.1 s + pieces (§5.5). + +Near and gap points come from the raw files and are placed at their real gps_time with step 1's rule: the last +trajectory node with t_node < t, if it is within 10 ms (step 1's own tolerance), so +`global = mrp[scan] · csv_pose[node] · calib · p_raw`. + +Placing step 1's own points by raw time as well is possible (§8) but measured slightly less sharp, so it is only an +experimental option. + +### 5.4 Near points + +Step 1 drops every point whose horizontal range is below 1.0 m, which leaves a hole around the walking path. Easy GUI +adds those points back at export; odometry never sees them. + +Rule: keep raw points with `near_min_range` ≤ 3D range **and** that step 1's distance filters removed. As implemented +this is the exact complement of step 1's filters: horizontal range ≤ `filter_threshold_xy_inner` (1.0 m) or 3D norm ≤ +`threshould_output_filter`, and horizontal range < `filter_threshold_xy_outer`. No-return points are always dropped. + +`near_min_range` defaults to 0.4 m (advanced setting "Closest points kept", 0.1–0.95 m). In measured raw data the +operator's body and the handle show up only as small blobs inside 0.4 m, and near points beyond that are environment in +every direction. + +**Output filter.** For the near rule to meet step 1's output without a gap, Easy GUI sets +`threshould_output_filter = filter_threshold_xy_inner` (1.0 m instead of the preset's 1.5 m). Because the output filter +is output only (§4), the trajectory is unchanged; this was confirmed byte for byte (§7.1). Points with horizontal range +≥ 1.0 m then all come from step 1, and points below come from the near rule. + +### 5.5 Gap points + +Step 1 takes `[first node, last node)` of each 0.1 s piece, so the raw points between a piece's last node and the next +piece's first node are never exported (about 5 % of the recording time). Easy GUI adds them back: a raw point inside +step 1's distance limits is a gap point when the last trajectory node at or before its time is the last node of a +piece, i.e. `j % threshold_nr_poses == threshold_nr_poses − 1` for node j within the scan. + +### 5.6 Never write into the recording + +The recording may be on the scanner's USB stick, so Easy GUI never writes into it. Because `load_data` writes `cache/` +and `preview/` next to its input files (§3), step 1 runs on a staging folder of symlinks inside the Easy GUI output +folder. The pre-flight check refuses an output folder inside the recording. + +### 5.7 Forestry preset + +The preset is `LidarOdometryParams` defaults plus the 18 overrides of "Presets → 2 … precise forestry" in +`lidar_odometry_gui.cpp`. HDMapping has no shared preset function (the values are inline GUI code, also copied into the +`precision_forestry` app), so `forestry_preset.cpp` repeats them and writes the TOML through step 1's own `TomlIO`. +The only deviation is the output filter of §5.4. + +## 6. The export + +Source: `src/export.cpp`. + +### 6.1 Flow and memory + +1. Load the session with `Session` in `load_cache_mode`: poses from `session_poses.mrp` plus the trajectories, no + points. This is exactly what step 2 uses. +2. Stream one `scan_lio_N.laz` at a time (step 1's points). +3. Stream one raw LAZ at a time (near and gap points). + +The distance thresholds are read from the session's own `HDMapping_params_*.toml`, i.e. what step 1 actually ran with. +Peak memory on the full test recording: 106 MB. + +### 6.2 Step 1's points + +These replicate step 2's `save_all_to_las`: `m_pose · (p − offset) + offset`, gps_time 0 skipped, scale 0.0001. The +result is integer-identical to `multi_view_tls_registration_step_2 --export` (§7.2). + +`range_m` of a step 1 point is `|calib⁻¹ · csv_pose[node]⁻¹ · p_local|`, with the node chosen by step 1's rule on the +ramp time. + +### 6.3 Point counts on the test recording + +10-minute forest walk, 115 file pairs, 115.9 M raw points of which 17.1 M are no-return records. + +| point_origin | Points | +|---|---| +| 0, step 1 | 89,326,756 | +| 1, near | 4,038,530 | +| 2, gap | 4,026,788 | +| total | 97.39 M | + +point_cloud.laz 948 MB, trajectory.csv 20 MB (114,940 poses). Export time 143 s. + +148 step 1 points (0.0002 %) have a range of 70–99.7 m. These are steep returns (elevation up to 54°): step 1's 70 m +limit is horizontal. + +### 6.4 Trajectory CSV + +One row per trajectory node: `timestamp_unix_s`, `x, y, z`, `qw, qx, qy, qz`, `om_deg, fi_deg, ka_deg` (HDMapping +Tait-Bryan angles), `scan_index` (= `point_source_id`), `host_clock_unix_s` (the IMU file's host clock, ms resolution, +not synchronised), and `imu_om_deg, imu_fi_deg, imu_ka_deg` (IMU-only orientation before odometry). Timestamps have +1 µs resolution. + +## 7. Validation + +`scripts/check_export.py [--step2 ]` automates the checks below (fields, range, alignment, gap +points, step 2 reference). It needs `laspy[lazrs]`, numpy and scipy. All numbers are from the test recording of §6.3. + +### 7.1 Easy GUI's step 1 run is the manual one + +Easy GUI (output filter 1.0 m) against a manual step 1 run with preset 2 (1.5 m): the trajectory is **byte-identical** +(`session_poses.mrp`, `trajectory_lio_*.csv`). So step 1 is deterministic here, and the output filter is confirmed to be +output only. `scan_lio` has 89.33 M points instead of 84.21 M (+5.11 M, +6.1 %), the points between 1.0 and 1.5 m. + +### 7.2 Step 1's points match step 2's export + +Against `multi_view_tls_registration_step_2 --export`: 89,326,756 points in both, max |ΔXYZ| = 0 (integer +coordinates identical), intensity and point_source_id identical. The trajectory CSV matches step 2's `trajectories.csv` +to 0.05 mm and 5e-10 in the quaternion. + +`range_m` of step 1's points, compared with the raw range on coincident pairs: p99.9 difference 0.07 mm. + +### 7.3 Near points line up with step 1's points + +Test mode `--check-band 1.2` also exports the raw points with horizontal range 1.0–1.2 m, which step 1 exported too, +placed the near-point way and flagged `point_origin = 3`. Their nearest-neighbour distance to step 1's points: + +- median 0.10 mm overall; +- for the 95.6 % that have a step 1 counterpart: p90 6.4 mm, p99 12.9 mm; 71 % land on the identical pose; +- the other 4.4 % lie in step 1's inter-chunk gaps, so they have no counterpart. + +Rebuilding step 1's ramp from the raw files for scans 40–49 reproduces all 7,595,561 step 1 points (≤ 0.17 mm) and +shows that the whole difference comes from step 1's ramp time choosing a different node than the real time would: +|ramp − raw| p50 1.2 ms, p99 11.6 ms, max 17 ms, giving displacements of p90 9 mm, max 56 mm. Displacements of this +size have a negligible effect on sharpness and stem diameters: placing all points by raw time moves them by the same +mechanism and changes ground and stem thickness by less than 1 mm and diameters by a median of 0.4 mm (§8). + +### 7.4 Gap points sit where a missing slice of step 1's points would + +Per scan, against step 1's points of scans s−1 … s+1, gap points have a nearest-neighbour distance of median 19.4 mm, +p99 187 mm. Control: step 1's own points from the 5 ms before each gap, queried without them: median 19.8 mm, p99 +192 mm. The same holds in every range bin (1–3 m: 10.6 vs 10.9 mm … 20+ m: 68.1 vs 68.2 mm). + +277 gap points are within 0.5 µs of a pose time; classifying points from trajectory.csv (1 µs resolution) needs a +1 µs tolerance for them. + +## 8. Experimental option: place every point by its raw time + +`place_by_raw_time` (GUI advanced checkbox, `--place-by-raw-time`; default **off**) skips `scan_lio` and places every +raw point inside step 1's distance limits from the raw files, with the last node before its real gps_time (within +10 ms), and writes the raw gps_time. The near rule is unchanged. + +On the test recording this gives 93,359,291 points inside step 1's limits instead of 89,326,756 (+4,026,808 from step +1's inter-chunk gaps, plus ~5.7 k first points of chunks). + +**Displacement against the default.** Same raw point in both versions (matched via the rebuilt ramp; both placements +verified against the files to ≤ 0.17 mm): the same pose for 79.3 % of points, median displacement 0 in every range bin. +p95 is 12 / 20 / 34 / 69 mm at 1–3 / 3–5 / 5–10 / 10–20 m (moved points only: median 8 / 13 / 23 / 47 mm, p95 +19 / 32 / 57 / 118 mm), which is a neighbouring node's rotation (~9 mrad). + +**Sharpness** (2026-10-17; the same 89,326,756 matched points in both versions, selections made on the mean of both +positions, so the comparison is paired): + +| Test | Default | Raw time | Paired difference (raw − default) by distance from the path | Raw time better in | +|---|---|---|---|---| +| Ground: 5,376 cells of 1 × 1 m within 20 m of the walk, lowest points in a ±0.15 m band, orthogonal plane fit, residual std | median 65.73 mm | 66.14 mm | +0.06 / +0.08 / +0.15 / +0.61 mm at 0–3 / 3–5 / 5–10 / 10–20 m | 37 % of cells (Wilcoxon p ≈ 1e-112) | +| Ground, ±0.05 m band | | | +0.05 … +0.24 mm | 41 % | +| Stems: 17 automatically detected, visually checked trunks, circle fit at 1.2–1.4 m above local ground, RMSE | median 19.25 mm | 19.31 mm | +0.05 / +0.43 / +0.03 / +0.17 mm at 1–3 / 3–5 / 5–10 / 10–20 m | 3 of 17 (p = 0.06) | + +The ~66 mm ground thickness is mostly forest-floor roughness, which the paired design cancels. Stem diameters do not +differ systematically: median |ΔD| 0.4 mm (< 2 mm within 10 m; −3 … +7 mm for one-sided arcs at 10–20 m). + +**Verdict:** the default (step 1's placement) is slightly sharper, by 0.1–0.6 mm on 20–60 mm thick surfaces. The most +likely reason is that step 1 optimised the trajectory with its own ramp-time points. Hence the option stays off. (The +ground and stem analysis scripts are not in the repository.) + +## 9. Resources: time, memory, disk + +Measured on the test recording (117 M raw points, 10-minute walk): + +- Step 1 takes ~21 min with a peak memory footprint of 8.5 GB; the export ~2.5 min (143 s) with ~106 MB. +- Step 1 needs ~108 B per raw point of temporary disk at its peak (cache 7.5 GB + chunk caches ~4 GB + `scan_lio` for + 117 M points). +- Results are 2.14× the raw LAZ size: session 0.82× + point_cloud.laz 1.29× (with near and gap points) + + trajectory.csv 0.03×. + +The pre-flight check therefore estimates 120 B per raw point of temporary space plus 2.25× the raw LAZ size for results, +always reports the breakdown, warns below 1.5× the estimate and refuses below 1×. + +## 10. Batch mode + +"Several recordings" in the window and `--batch ` process many recordings one after another, +for example the plots of a field day overnight. Each recording runs through the unchanged `Pipeline` (pre-flight +re-check, staging, step 1 as a child process, export, clean-up), so its output folder is exactly that of a single run. + +``` +/ + batch.log what happened, with times (flushed per line) + batch_summary.csv one row per recording and per skipped folder, rewritten after every recording + / one per recording: the files of §2 (+ easy_gui_done.txt once finished) +``` + +### 10.1 Finding recordings + +`discoverRecordings` looks at the parent's direct subfolders, plus the parent itself (listed first, so a single +recording folder also works). A recording is a folder in which `classifyRecordingFiles` finds at least one point cloud +and one IMU file; the pre-flight check judges the rest. It is deliberately not recursive: a recording is the folder the +scanner wrote, and a deeper search would also find step 1's own `cache/`/`preview/` folders of old manual runs and +Easy GUI's staging links. Other subfolders are listed with a reason ("No IMU files", "Easy Forest HDMapping results, +not a recording", "Recordings are only looked for one folder level down"). Hidden and system folders (`.*`, `$*`, +`System Volume Information`) are left out silently. + +### 10.2 Output names + +The default is the recording's folder name, made unique ignoring ASCII case (the default file systems of macOS and +Windows are case-insensitive) by appending `_2`, `_3`, ... A parent that is a recording and contains a recording of +the same name is the common collision. A name is also passed over when its folder already holds files other than this +recording's finished result or an unfinished attempt. Edited names are validated for both systems (no +`/ \ : * ? " < > |` or control characters, no trailing dot or space, not `.`/`..`, not a Windows device name such as +`CON` or `COM1`) and must be unique among the recordings that will run. + +The default output root is `Documents/Easy Forest HDMapping/`. Unlike single mode it is reused when +it holds an earlier batch (it contains `batch_summary.csv`), so that choosing the same parent folder again continues +the batch. + +The batch refuses an output root inside any recording (`batchOutputProblem`): `batch.log` and the summary would be +written there. The recordings' own output folders are covered by the per-recording check (§5.6). + +### 10.3 Checks before the start + +Every recording gets the single-mode pre-flight check against its own output folder, row by row on a background thread +in the window (the recordings may be on a slow USB stick). A recording that fails cannot be selected; one that is +already done (§10.4) is shown as such and skipped. The batch disk check needs + + the largest temporary need of any one selected recording + the results of all selected recordings, + +because the recordings run one at a time and each frees its temporary space (§9) when it ends. Thresholds are those +of single mode: refused below the estimate, a warning below 1.5×. Disk space is checked again before every recording +(the pipeline repeats its pre-flight check), so a disk that fills up during the batch fails the affected recordings +with the usual message instead of crashing step 1. + +### 10.4 Failures, done markers and restarting + +- A recording that fails (check, step 1 or export) is marked failed with the pipeline's plain-language message, and the + batch continues. Its output folder keeps what single mode keeps after a failure: the log, and the step 1 session if + only the export failed. +- A recording that succeeds gets `easy_gui_done.txt`: version, recording path, a fingerprint, finish time, duration, + raw and exported point counts. The fingerprint is the number of file pairs, the total size of the paired files and + the first IMU timestamp. It identifies the recording without its path, because a USB stick can be mounted under + another name or drive letter after a restart; the IMU timestamp separates equally large recordings. +- Restarting the same batch (same parent, same output root) skips recordings whose marker matches. Their summary rows + say `already_done` and take duration and point counts from the marker. +- A recording whose output folder holds only Easy GUI's own run files (`easy_gui.log`, `work/`, `point_cloud.laz`, + `trajectory.csv`, `README.txt`) but no marker is an unfinished attempt (failed, stopped, power cut). The batch runs + the pipeline with `PipelineRequest::resume_unfinished`: the pre-flight check accepts the folder + (`runPreflight(..., resume_unfinished)`), and the pipeline removes everything in it except `easy_gui.log`, which it + continues. Any other file in the folder makes the check fail ("already contains files"), so nothing a user put + there is ever deleted. Single mode keeps refusing non-empty folders. +- The summary is written to `batch_summary.csv.tmp` and renamed, after every recording, so it is never half-written + and is current even if the batch never ends normally. On Windows a summary that is open in Excel cannot be + replaced; the batch logs a warning and continues. + +Summary columns: `recording, output_name, status, duration_s, raw_points, exported_points, output_path, error`. +`raw_points` counts all point records of the paired files (including no-return records), as the pre-flight check +does; `exported_points` is the size of `point_cloud.laz`. Statuses: `succeeded`, `failed`, `already_done`, +`cancelled` (stopped by "stop now"), `not_run` (the batch stopped before it), `excluded` (not selected, or its check +failed; the error column says which) and `skipped` (not a recording). + +### 10.5 Running + +- **Stop after current** lets the running recording finish; the rest become `not_run`. **Stop now** cancels the + running recording exactly like single mode's cancel (child process killed, `work/` removed, log kept). +- **Sleep.** The batch thread holds a `SleepBlocker` for its whole run. On macOS this is an IOKit assertion + `kIOPMAssertionTypePreventUserIdleSystemSleep` (shows up in `pmset -g assertions` as "Easy Forest HDMapping is + processing recordings"; checked, and released when the batch ends); on Windows + `SetThreadExecutionState(ES_CONTINUOUS | ES_SYSTEM_REQUIRED)`, which applies to the calling thread, so it is set and cleared on the batch thread. Both + prevent idle sleep only: the display may turn off, and closing a laptop lid still sleeps. Other platforms: no-op. +- **Progress and ETA.** Each recording is weighted by its raw point count. Within a recording, odometry counts for + 90 % and the export for 10 % (§9: ~21 min against ~2.5 min). The ETA is the processing rate of the recordings that + really ran (already-done recordings take no time) applied to the remaining weight. +- `BatchRunner` drives an abstract `RecordingJob`; the real one wraps `Pipeline`, and the tests inject a fake one. + +### 10.6 Tests + +Unit tests (`tests/test_batch.cpp`, fake `RecordingJob`): discovery (sorting, nested folders, non-recordings, hidden +folders, parent that is a recording, with and without a trailing separator), output-name validation and collisions +(case, invalid characters, foreign / unfinished / finished folders), done-marker round trip and path independence, +batch disk arithmetic, output root inside a recording, failure-continues with summary and log contents, restart +skipping finished recordings and retrying an unfinished one, a folder filled after the check, stop after current and +stop now. `tests/test_preflight.cpp` covers `resume_unfinished` and keeps `holdsUnfinishedOutput` in sync with +`OutputLayout`. + +Integration test (`EASY_GUI_TEST_RECORDING=mini4`): a parent with two copies of the recording (symlinked files), a +recording that passes the check but on which step 1 fails (one file pair cut to 200 points), and a non-recording +folder. Both copies succeed with identical point counts (2,566,166), the broken one fails with step 1's message, and a +restart skips the two copies and retries the broken one. 62 s on an Apple silicon laptop; one `mini4` copy takes +~30 s. + +## 11. Status and next steps + +- **M1 (done):** pre-flight check, staging, step 1 as a child process with progress and cancel. +- **M2 (done):** export (§6). +- **M3 (in progress):** validation script. `check_export.py` covers fields, range, alignment, gap points and the step 2 + reference. Still open: + - Ramp-time error: for each odometry chunk, rebuild step 1's ramp from the raw points and compare with their real + gps_time, reported twice: as step 1 does it (no-return points removed by the load filter *before* the ramp) and + with those points kept, to quantify how much their removal contributes. + - Near points: counts, range histogram, and sensor-frame density below 1.5 m. +- **M4 (done, released as 0.2.0):** batch mode (§10). The macOS side was run in the window and with + `--batch` on `mini4` copies (stop after current, restart, sleep assertion). The Windows code (`SetThreadExecutionState`, + Ctrl+C in `--batch`) compiles and its unit tests pass in CI (run 39163046329); not yet run on a real Windows PC. +- **v2:** gated pose-graph loop closure (§1). +- **Release builds:** `.github/workflows/easy-forest-{macos,windows}.yml`, on `easy-forest-v*` tags and manual + dispatch; the packaging steps are scripts in `apps/easy_gui/packaging/` so they run locally the same way. + - macOS (`macos-15` runner, Apple silicon, minimum 13.0): PROJ is built from source as a static library for 13.0 + (`build_proj.sh`; core_math links it for GNSS, which Easy GUI never calls; Homebrew's is built for the runner's + macOS and drags in libtiff/curl). oneTBB without hwloc (`TBB_DISABLE_HWLOC_AUTOMATIC_SEARCH=ON`). OpenCV is not + configured at all in `HDMAPPING_EASY_GUI_ONLY` builds (only the camera apps link it). `make_app.py` builds + `Easy Forest HDMapping.app` (`easy_gui` main executable, `lidar_odometry_step_1` next to it in `Contents/MacOS`, + `liblaszip`, `libtbb`, `libglut` in `Contents/Frameworks`, rpaths rewritten), re-signs it ad hoc and fails unless + every bundled binary has minos <= 13.0 and references nothing outside the bundle, `/usr/lib` and `/System`. The + build also fails on the linker's "built for newer macOS version" warning, which a static library would only show + there. `make_dmg.sh` adds the Applications shortcut. Local and CI run: app 7.9 MB, DMG 3.6 MB; locally the bundled app + processed `mini4` headless. + - Windows (`windows-latest`, x64): `easy_gui.exe` is a WIN32-subsystem program (`/ENTRY:mainCRTStartup`, so `main` + stays), `--headless`/`--export-only` attach to the parent console. A manifest sets UTF-8 as the process code page, + because the file dialog, drag and drop and `ChildProcess` all use UTF-8. `HD_CPU_OPTIMIZATION=GENERIC`: `AUTO` + adds `/arch:AVX2` from the runner's CPU. `make_zip.py` collects the DLLs with `dumpbin /dependents`, copies the + MSVC runtime app-locally from VS's redist folder and fails on any DLL that is neither bundled nor part of Windows. + First CI run: folder 7.1 MB (two exes, `laszip3`, `freeglut`, `tbb12` and the MSVC runtime DLLs), zip 3.3 MB. + PROJ's DLL is not needed: nothing Easy GUI calls references it, so the linker drops the import. + - `easy_gui_tests` cover `ChildProcess` on Windows too (cmd.exe; argument quoting through `--echo-args`). + - No app icon yet (none exists for Easy GUI); no code signing or notarization (README explains the warnings). + +Tests: `easy_gui_tests` (doctest, synthetic recordings and sessions). The integration tests against the real step 1 +binary run only when `EASY_GUI_TEST_RECORDING=` is set; a folder of symlinks to the first 4 file +pairs of a recording is enough. diff --git a/apps/easy_gui/docs/images/easy_gui_batch.png b/apps/easy_gui/docs/images/easy_gui_batch.png new file mode 100644 index 00000000..56767044 Binary files /dev/null and b/apps/easy_gui/docs/images/easy_gui_batch.png differ diff --git a/apps/easy_gui/docs/images/easy_gui_batch_running.png b/apps/easy_gui/docs/images/easy_gui_batch_running.png new file mode 100644 index 00000000..953024b0 Binary files /dev/null and b/apps/easy_gui/docs/images/easy_gui_batch_running.png differ diff --git a/apps/easy_gui/docs/images/easy_gui_window.png b/apps/easy_gui/docs/images/easy_gui_window.png new file mode 100644 index 00000000..64a33ffd Binary files /dev/null and b/apps/easy_gui/docs/images/easy_gui_window.png differ diff --git a/apps/easy_gui/packaging/macos/build_proj.sh b/apps/easy_gui/packaging/macos/build_proj.sh new file mode 100755 index 00000000..d7c5dc86 --- /dev/null +++ b/apps/easy_gui/packaging/macos/build_proj.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# Builds a static PROJ for the Easy Forest HDMapping macOS release. +# +# Usage: build_proj.sh [work dir] +# +# Why: core_math links PROJ::proj (core/src/gnss.cpp, GNSS georeferencing, which Easy GUI never calls), so every +# build needs one. Homebrew's libproj is built for the runner's own macOS version (and pulls in libtiff/curl/sqlite +# dylibs built the same way), so a bundle using it would not start on older macOS. A static PROJ built here for the +# deployment target leaves nothing to bundle: only the system libsqlite3 and libc++ are linked dynamically. +# TIFF (grids) and curl (network grids) are off: neither is needed to compile/link core_math. +set -euo pipefail + +PROJ_VERSION=9.6.2 +PROJ_SHA256=53d0cafaee3bb2390264a38668ed31d90787de05e71378ad7a8f35bb34c575d1 +DEPLOYMENT_TARGET="${MACOSX_DEPLOYMENT_TARGET:-13.0}" + +prefix="${1:?usage: build_proj.sh [work dir]}" +work="${2:-$(mktemp -d)}" +mkdir -p "$work" +prefix="$(cd "$(dirname "$prefix")" && pwd)/$(basename "$prefix")" + +tarball="$work/proj-$PROJ_VERSION.tar.gz" +if [ ! -f "$tarball" ]; then + curl -sSLf -o "$tarball" "https://github.com/OSGeo/PROJ/releases/download/$PROJ_VERSION/proj-$PROJ_VERSION.tar.gz" +fi +echo "$PROJ_SHA256 $tarball" | shasum -a 256 -c - + +rm -rf "$work/proj-$PROJ_VERSION" "$work/build" +tar -xzf "$tarball" -C "$work" + +cmake -S "$work/proj-$PROJ_VERSION" -B "$work/build" \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX="$prefix" \ + -DCMAKE_OSX_DEPLOYMENT_TARGET="$DEPLOYMENT_TARGET" \ + -DBUILD_SHARED_LIBS=OFF \ + -DBUILD_TESTING=OFF \ + -DBUILD_APPS=OFF \ + -DENABLE_TIFF=OFF \ + -DENABLE_CURL=OFF \ + -DEMBED_PROJ_DATA_PATH=OFF +cmake --build "$work/build" --config Release -j"$(sysctl -n hw.ncpu)" +cmake --install "$work/build" --config Release + +echo "PROJ $PROJ_VERSION installed to $prefix (macOS $DEPLOYMENT_TARGET, static)" diff --git a/apps/easy_gui/packaging/macos/make_app.py b/apps/easy_gui/packaging/macos/make_app.py new file mode 100755 index 00000000..62228d13 --- /dev/null +++ b/apps/easy_gui/packaging/macos/make_app.py @@ -0,0 +1,208 @@ +#!/usr/bin/env python3 +"""Builds "Easy Forest HDMapping.app" from a build/bin directory. + +Usage: make_app.py [--version X.Y.Z] [--min-macos 13.0] + +Layout: + Contents/MacOS/easy_gui main executable (CFBundleExecutable) + Contents/MacOS/lidar_odometry_step_1 started by easy_gui as a child process from its own directory + Contents/Frameworks/*.dylib every non-system dylib either executable needs, recursively + +Every bundled Mach-O refers to its bundled dependencies as @rpath/; the executables get the single rpath +@executable_path/../Frameworks and the dylibs @loader_path, so nothing points back into the build machine. The bundle +is then re-signed ad hoc (path rewriting invalidates the linker's signature, and arm64 refuses to run unsigned code) +and checked: every bundled binary must have minos <= --min-macos and may only reference the bundle, /usr/lib and +/System. Any violation fails the script, so a CI build cannot ship a bundle that does not start on the target macOS. +""" + +import argparse +import os +import plistlib +import re +import shutil +import subprocess +import sys +from pathlib import Path + +APP_NAME = "Easy Forest HDMapping" +BUNDLE_ID = "io.github.martinmokros.easyforesthdmapping" +EXECUTABLES = ["easy_gui", "lidar_odometry_step_1"] +SYSTEM_PREFIXES = ("/usr/lib/", "/System/") + + +def run(*args): + return subprocess.run(args, check=True, capture_output=True, text=True).stdout + + +def is_macho(path): + with open(path, "rb") as f: + magic = f.read(4) + return magic in (b"\xcf\xfa\xed\xfe", b"\xce\xfa\xed\xfe", b"\xca\xfe\xba\xbe") + + +def load_commands(path): + """(dependencies, rpaths, dylib id, minimum macOS) from otool -l.""" + deps, rpaths, dylib_id, minos = [], [], None, None + cmd = None + for line in run("otool", "-l", str(path)).splitlines(): + line = line.strip() + if line.startswith("cmd "): + cmd = line.split()[1] + elif line.startswith("name ") and cmd in ("LC_LOAD_DYLIB", "LC_LOAD_WEAK_DYLIB", "LC_REEXPORT_DYLIB"): + deps.append(line[5:].rsplit(" (offset", 1)[0]) + elif line.startswith("name ") and cmd == "LC_ID_DYLIB": + dylib_id = line[5:].rsplit(" (offset", 1)[0] + elif line.startswith("path ") and cmd == "LC_RPATH": + rpaths.append(line[5:].rsplit(" (offset", 1)[0]) + elif line.startswith("minos ") and cmd == "LC_BUILD_VERSION": + minos = line.split()[1] + elif line.startswith("version ") and cmd == "LC_VERSION_MIN_MACOSX": + minos = line.split()[1] + return deps, rpaths, dylib_id, minos + + +def expand(path, loader, executable): + return Path(path.replace("@loader_path", str(loader.parent)).replace("@executable_path", str(executable.parent))) + + +def resolve(dep, macho, rpaths, executable): + """Where the build-tree binary `macho` really loads `dep` from.""" + if dep.startswith("@rpath/"): + for rpath in rpaths: + candidate = expand(rpath, macho, executable) / dep[len("@rpath/"):] + if candidate.exists(): + return candidate.resolve() + return None + candidate = expand(dep, macho, executable) + return candidate.resolve() if candidate.exists() else None + + +def version_tuple(v): + return tuple(int(x) for x in v.split(".")) + + +def bundle(bin_dir, out_dir, version, min_macos): + app = out_dir / f"{APP_NAME}.app" + if app.exists(): + shutil.rmtree(app) + macos_dir = app / "Contents" / "MacOS" + frameworks = app / "Contents" / "Frameworks" + resources = app / "Contents" / "Resources" + for d in (macos_dir, frameworks, resources): + d.mkdir(parents=True) + + # Source binary -> bundled copy, walked breadth first so each dylib is resolved with the rpaths of the build-tree + # binary that loads it. + queue = [] + for name in EXECUTABLES: + src = bin_dir / name + if not src.is_file(): + sys.exit(f"error: {src} not found") + dst = macos_dir / name + shutil.copy2(src, dst) + queue.append((src, dst, src)) + + bundled = {} # resolved source path -> bundled name + while queue: + src, dst, executable = queue.pop(0) + deps, rpaths, _, _ = load_commands(src) + for dep in deps: + if dep.startswith(SYSTEM_PREFIXES): + continue + real = resolve(dep, src, rpaths, executable) + if real is None: + sys.exit(f"error: {src.name} needs {dep}, which cannot be found") + name = Path(dep).name + if real not in bundled: + if any(n == name for n in bundled.values()): + sys.exit(f"error: two different dylibs are both called {name}") + bundled[real] = name + lib_dst = frameworks / name + shutil.copy2(real, lib_dst) + lib_dst.chmod(0o755) + queue.append((real, lib_dst, executable)) + print(f" bundling {real} as Frameworks/{name}") + if dep != f"@rpath/{bundled[real]}": + run("install_name_tool", "-change", dep, f"@rpath/{bundled[real]}", str(dst)) + for rpath in rpaths: + run("install_name_tool", "-delete_rpath", rpath, str(dst)) + new_rpath = "@executable_path/../Frameworks" if dst.parent == macos_dir else "@loader_path" + run("install_name_tool", "-add_rpath", new_rpath, str(dst)) + if dst.parent == frameworks: + run("install_name_tool", "-id", f"@rpath/{dst.name}", str(dst)) + + with open(app / "Contents" / "Info.plist", "wb") as f: + plistlib.dump( + { + "CFBundleDevelopmentRegion": "en", + "CFBundleDisplayName": APP_NAME, + "CFBundleName": APP_NAME, + "CFBundleExecutable": "easy_gui", + "CFBundleIdentifier": BUNDLE_ID, + "CFBundleInfoDictionaryVersion": "6.0", + "CFBundlePackageType": "APPL", + "CFBundleShortVersionString": version, + "CFBundleVersion": version, + "LSMinimumSystemVersion": min_macos, + "LSApplicationCategoryType": "public.app-category.utilities", + "NSHighResolutionCapable": True, + "NSHumanReadableCopyright": "MIT License. Built on HDMapping by Janusz Będkowski and the MapsHD contributors.", + }, + f, + ) + + # Only after all rewriting: install_name_tool invalidates signatures. + run("codesign", "--force", "--deep", "-s", "-", str(app)) + return app + + +def verify(app, min_macos): + errors = [] + machos = [p for p in app.rglob("*") if p.is_file() and not p.is_symlink() and is_macho(p)] + frameworks = app / "Contents" / "Frameworks" + print(f"Checking {len(machos)} Mach-O files (minos <= {min_macos}):") + for p in sorted(machos): + deps, rpaths, _, minos = load_commands(p) + rel = p.relative_to(app) + print(f" {rel}: minos {minos}") + if minos is None: + errors.append(f"{rel}: no minimum macOS version in the load commands") + elif version_tuple(minos) > version_tuple(min_macos): + errors.append(f"{rel}: minos {minos} > {min_macos}") + for rpath in rpaths: + if not rpath.startswith(("@executable_path", "@loader_path")): + errors.append(f"{rel}: rpath {rpath} points outside the bundle") + for dep in deps: + if dep.startswith(SYSTEM_PREFIXES): + continue + if not dep.startswith("@rpath/") or not (frameworks / dep[len("@rpath/"):]).is_file(): + errors.append(f"{rel}: dependency {dep} is not in Contents/Frameworks") + try: + run("codesign", "--verify", "--deep", "--strict", str(app)) + except subprocess.CalledProcessError as e: + errors.append(f"codesign --verify failed: {e.stderr.strip()}") + return errors + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("bin_dir", type=Path) + parser.add_argument("out_dir", type=Path) + parser.add_argument("--version", default="0.0.0") + parser.add_argument("--min-macos", default="13.0") + args = parser.parse_args() + + version = re.sub(r"^[^0-9]*", "", args.version) or "0.0.0" + args.out_dir.mkdir(parents=True, exist_ok=True) + app = bundle(args.bin_dir.resolve(), args.out_dir.resolve(), version, args.min_macos) + errors = verify(app, args.min_macos) + if errors: + print("\nBundle check FAILED:", file=sys.stderr) + for e in errors: + print(f" {e}", file=sys.stderr) + sys.exit(1) + print(f"\n{app} OK") + + +if __name__ == "__main__": + main() diff --git a/apps/easy_gui/packaging/macos/make_dmg.sh b/apps/easy_gui/packaging/macos/make_dmg.sh new file mode 100755 index 00000000..cec64205 --- /dev/null +++ b/apps/easy_gui/packaging/macos/make_dmg.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# Packs "Easy Forest HDMapping.app" into a DMG whose window shows the app next to an Applications shortcut, so +# installing is one drag (README: Installation -> Mac). +# +# Usage: make_dmg.sh +set -euo pipefail + +app="${1:?usage: make_dmg.sh }" +dmg="${2:?usage: make_dmg.sh }" + +staging="$(mktemp -d)" +trap 'rm -rf "$staging"' EXIT + +# ditto keeps the bundle's symlinks, permissions and signature intact. +ditto "$app" "$staging/$(basename "$app")" +ln -s /Applications "$staging/Applications" + +rm -f "$dmg" +hdiutil create -volname "Easy Forest HDMapping" -srcfolder "$staging" -fs HFS+ -format UDZO -ov "$dmg" +hdiutil verify "$dmg" diff --git a/apps/easy_gui/packaging/windows/make_zip.py b/apps/easy_gui/packaging/windows/make_zip.py new file mode 100755 index 00000000..f95dbac1 --- /dev/null +++ b/apps/easy_gui/packaging/windows/make_zip.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +"""Builds the Easy Forest HDMapping Windows folder and zip from a build/bin/Release directory. + +Usage: make_zip.py [--search-dir DIR ...] + +The folder holds easy_gui.exe, lidar_odometry_step_1.exe (started by easy_gui from its own directory) and every DLL +they need, found recursively with `dumpbin /dependents`. The MSVC runtime (msvcp140, vcruntime140, ...) is copied +app-locally from Visual Studio's redist directory: the build runner has it installed system-wide, a user's PC may not. +Everything else must be part of Windows (api-ms-win-*, or present in System32 and not a VC runtime DLL); a DLL that +can be found neither in the build nor in --search-dir fails the script. The zip contains the files at its top level +(README: extract into a folder, double-click easy_gui.exe). +""" + +import argparse +import glob +import os +import re +import shutil +import subprocess +import sys +import zipfile +from pathlib import Path + +EXECUTABLES = ["easy_gui.exe", "lidar_odometry_step_1.exe"] +VC_RUNTIME = re.compile(r"^(msvcp\d+.*|vcruntime\d+.*|concrt\d+|vcomp\d+|vccorlib\d+)\.dll$", re.IGNORECASE) +SYSTEM32 = Path(os.environ.get("SystemRoot", r"C:\Windows")) / "System32" + + +def vswhere(*args): + exe = Path(os.environ.get("ProgramFiles(x86)", r"C:\Program Files (x86)")) / "Microsoft Visual Studio/Installer/vswhere.exe" + return subprocess.run([str(exe), "-latest", "-products", "*", *args], check=True, capture_output=True, text=True).stdout.strip() + + +def find_tools(): + vs = Path(vswhere("-property", "installationPath")) + dumpbin = sorted(glob.glob(str(vs / "VC/Tools/MSVC/*/bin/Hostx64/x64/dumpbin.exe"))) + redist = sorted(glob.glob(str(vs / "VC/Redist/MSVC/*/x64"))) + if not dumpbin or not redist: + sys.exit(f"error: dumpbin or the VC redist directory not found under {vs}") + # Highest version wins; the redist folder holds Microsoft.VC14x.CRT (and .OpenMP) subfolders. + crt_dirs = [Path(d) for d in glob.glob(str(Path(redist[-1]) / "Microsoft.VC*.CRT"))] + crt_dirs += [Path(d) for d in glob.glob(str(Path(redist[-1]) / "Microsoft.VC*.OpenMP"))] + return Path(dumpbin[-1]), crt_dirs + + +def dependents(dumpbin, binary): + out = subprocess.run([str(dumpbin), "/nologo", "/dependents", str(binary)], check=True, capture_output=True, text=True).stdout + return [line.strip() for line in out.splitlines() if line.strip().lower().endswith(".dll") and " " not in line.strip()] + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("bin_dir", type=Path) + parser.add_argument("out_dir", type=Path) + parser.add_argument("zip", type=Path) + parser.add_argument("--search-dir", type=Path, action="append", default=[]) + args = parser.parse_args() + + dumpbin, crt_dirs = find_tools() + print(f"dumpbin: {dumpbin}\nVC runtime: {', '.join(map(str, crt_dirs))}") + + if args.out_dir.exists(): + shutil.rmtree(args.out_dir) + args.out_dir.mkdir(parents=True) + + queue = [] + for name in EXECUTABLES: + src = args.bin_dir / name + if not src.is_file(): + sys.exit(f"error: {src} not found") + shutil.copy2(src, args.out_dir / name) + queue.append(args.out_dir / name) + + search = [args.bin_dir, *args.search_dir] + copied, system, errors = set(), set(), [] + while queue: + binary = queue.pop(0) + for dll in dependents(dumpbin, binary): + key = dll.lower() + if key in copied or key in system: + continue + if key.startswith(("api-ms-win-", "ext-ms-")): + system.add(key) + continue + if VC_RUNTIME.match(dll): + source = next((d / dll for d in crt_dirs if (d / dll).is_file()), None) + else: + source = next((d / dll for d in search if (d / dll).is_file()), None) + if source is None and (SYSTEM32 / dll).is_file(): + system.add(key) + continue + if source is None: + errors.append(f"{binary.name} needs {dll}, which is neither in the build, a search dir nor Windows") + continue + shutil.copy2(source, args.out_dir / source.name) + copied.add(key) + print(f" {binary.name} -> {dll} (from {source.parent})") + queue.append(args.out_dir / source.name) + + # The C++ runtime DLLs load each other dynamically in places (e.g. msvcp140 -> msvcp140_1/_2 for some + # features), so ship the whole CRT folder rather than only what dumpbin lists. + for d in crt_dirs: + if d.name.endswith(".CRT"): + for dll in d.glob("*.dll"): + if dll.name.lower() not in copied: + shutil.copy2(dll, args.out_dir / dll.name) + copied.add(dll.name.lower()) + + if errors: + print("\nPackage check FAILED:", file=sys.stderr) + for e in errors: + print(f" {e}", file=sys.stderr) + sys.exit(1) + + print(f"\nSystem DLLs (not bundled): {', '.join(sorted(system))}") + print(f"Folder contents:") + for f in sorted(args.out_dir.iterdir()): + print(f" {f.name:40s} {f.stat().st_size / 1e6:8.2f} MB") + + args.zip.unlink(missing_ok=True) + with zipfile.ZipFile(args.zip, "w", zipfile.ZIP_DEFLATED, compresslevel=9) as z: + for f in sorted(args.out_dir.iterdir()): + z.write(f, f.name) + print(f"\n{args.zip}: {args.zip.stat().st_size / 1e6:.1f} MB") + + +if __name__ == "__main__": + main() diff --git a/apps/easy_gui/scripts/check_export.py b/apps/easy_gui/scripts/check_export.py new file mode 100644 index 00000000..b78eb59b --- /dev/null +++ b/apps/easy_gui/scripts/check_export.py @@ -0,0 +1,275 @@ +#!/usr/bin/env python3 +"""Acceptance checks for an Easy GUI export (point_cloud.laz + trajectory.csv). + + check_export.py [--step2 ] + +Needs laspy with a LAZ backend, numpy and scipy (pip install "laspy[lazrs]" numpy scipy). + +Checks: + fields LAS 1.4 / PF6, GPS time type bit, named extra bytes (range_m float32, point_origin uint8), + point_source_id = scan index, user_data = lidar id, gps_time is Unix seconds. + range histogram of range_m; nothing below the near limit, nothing at the sensor origin. + alignment only with an export made with --check-band: nearest-neighbour distance from each check-band point + (point_origin 3: raw point placed the near-point way) to step 1's own points. A few mm median = same + transform chain. + gap points nearest-neighbour distance from each gap point (point_origin 2) to step 1's points of the same and the + neighbouring scans, next to a control: step 1's own points from the 5 ms before each gap, queried against + step 1's points without them (a correctly placed missing slice). + reference with --step2: the point_origin = 0 points against step 2's full-resolution export of the same session + (multi_view_tls_registration_step_2 --export ): count, XYZ, intensity, trajectory. +Exit code 1 if a check fails. +""" + +import argparse +import sys +from pathlib import Path + +import laspy +import numpy as np +from scipy.spatial import cKDTree + +CHUNK = 5_000_000 +failures = [] + + +def check(ok, message): + print((" ok " if ok else " FAIL ") + message) + if not ok: + failures.append(message) + + +def check_fields(laz): + print("fields") + with laspy.open(laz) as f: + h = f.header + check(str(h.version) == "1.4" and h.point_format.id == 6, f"LAS {h.version}, point format {h.point_format.id}") + check(not h.global_encoding.gps_time_type, "global encoding bit 0 (GPS time type) = 0") + extra = {d.name: d for d in h.point_format.extra_dimensions} + check(set(extra) == {"range_m", "point_origin"}, f"extra bytes: {sorted(extra)}") + check(np.dtype(extra["range_m"].dtype) == np.float32, f"range_m is {np.dtype(extra['range_m'].dtype)}") + check(np.dtype(extra["point_origin"].dtype) == np.uint8, f"point_origin is {np.dtype(extra['point_origin'].dtype)}") + print(f" {h.point_count:,} points, bounds {h.mins.round(1)} .. {h.maxs.round(1)}, {h.generating_software}") + + +def scan(laz): + """One streaming pass: per-flag counts, range histogram, sensor-origin and time checks, and the points needed for + the alignment check (step 1 points up to 3 m + check-band points), all with bounded memory. 3 m, not the band's + 1.2 m: the band limits the horizontal range, steep points reach ~2 m in 3D.""" + stats = {"count": np.zeros(4, np.int64), "hist": np.zeros(1201, np.int64), "range_min": [np.inf] * 4, + "range_max": [-np.inf] * 4, "t_min": np.inf, "t_max": -np.inf, "psid": set(), "user_data": set(), + "origin": 0, "close_odo": [], "band": [], "big_range": []} + with laspy.open(laz) as f: + for p in f.chunk_iterator(CHUNK): + flag = np.asarray(p.point_origin) + r = np.asarray(p.range_m) + xyz = np.column_stack([p.x, p.y, p.z]) + for k in range(4): + m = flag == k + if m.any(): + stats["count"][k] += m.sum() + stats["range_min"][k] = min(stats["range_min"][k], float(r[m].min())) + stats["range_max"][k] = max(stats["range_max"][k], float(r[m].max())) + stats["hist"] += np.bincount(np.minimum((r / 0.1).astype(np.int64), 1200), minlength=1201) + t = np.asarray(p.gps_time) + stats["t_min"], stats["t_max"] = min(stats["t_min"], t.min()), max(stats["t_max"], t.max()) + stats["psid"].update(np.unique(np.asarray(p.point_source_id)).tolist()) + stats["user_data"].update(np.unique(np.asarray(p.user_data)).tolist()) + stats["origin"] += int((np.abs(xyz).sum(axis=1) == 0).sum()) + int((r == 0).sum()) + m = (flag == 0) & (r < 3.0) + stats["close_odo"].append((xyz[m], r[m])) + stats["band"].append(xyz[flag == 3]) + far = r > 70 + if far.any(): + stats["big_range"].append(np.column_stack([r[far], xyz[far, 2], np.asarray(p.point_source_id)[far]])) + return stats + + +def check_range(stats, near_min): + print("range") + names = ["step 1 (0)", "near point (1)", "gap point (2)", "check band (3)"] + for k in range(4): + if stats["count"][k]: + print(f" {names[k]:15s} {stats['count'][k]:>12,} points, range_m {stats['range_min'][k]:.3f} .. {stats['range_max'][k]:.3f} m") + check(min(stats["range_min"]) >= near_min - 1e-6, f"no range_m below {near_min} m (min {min(stats['range_min']):.4f})") + check(stats["origin"] == 0, f"no points at (0,0,0) or with range 0 ({stats['origin']})") + h = stats["hist"] + edges = [0, 0.4, 1.0, 1.5, 3, 10, 30, 50, 70, 80, 100, 120.1] + print(" range_m histogram:") + for a, b in zip(edges[:-1], edges[1:]): + n = h[int(round(a * 10)):int(round(b * 10))].sum() + print(f" {a:6.1f} - {b:6.1f} m {n:>12,} {100 * n / h.sum():6.2f} %") + beyond = h[700:].sum() + print(f" beyond 70 m: {beyond:,} points ({100 * beyond / h.sum():.4f} %)") + if stats["big_range"]: + # step 1's 70 m limit is horizontal, in the lidar frame: steep returns can be further away in 3D + big = np.concatenate(stats["big_range"]) + print(f" (step 1 limits the horizontal range only; these are in {len(np.unique(big[:, 2]))} scans, " + f"z {big[:, 1].min():.1f} .. {big[:, 1].max():.1f} m)") + + +def check_alignment(stats): + band = np.concatenate(stats["band"]) if stats["band"] else np.empty((0, 3)) + if len(band) == 0: + return + print("alignment (check-band points vs step 1 points)") + band_r_max = stats["range_max"][3] + odo = np.concatenate([o[0][o[1] <= band_r_max + 0.01] for o in stats["close_odo"]]) + d_mm = cKDTree(odo).query(band, k=1, workers=-1)[0] * 1000 + q = np.percentile(d_mm, [50, 90, 99]) + print(f" {len(band):,} check-band points vs {len(odo):,} step 1 points up to {band_r_max:.2f} m") + print(f" NN distance: median {q[0]:.2f} mm, p90 {q[1]:.2f}, p99 {q[2]:.2f}, max {d_mm.max():.1f} mm") + print(f" exactly coincident (< 0.15 mm, same trajectory pose as step 1): {100 * (d_mm < 0.15).mean():.1f} %") + check(q[0] < 5.0, f"median NN distance {q[0]:.2f} mm < 5 mm") + + +def check_gaps(laz, folder): + """Gap points vs step 1's points, per scan (tree: step 1 points of scans s-1..s+1), next to a control: step 1's + points from the last node interval before each gap ([node 18, node 19) of every 20-node piece), queried against the + same tree without them.""" + print("gap points (vs step 1 points)") + parts = [] + with laspy.open(laz) as f: + for p in f.chunk_iterator(CHUNK): + o = np.asarray(p.point_origin) + m = (o == 0) | (o == 2) + parts.append((np.column_stack([p.x[m], p.y[m], p.z[m]]).astype(np.float32), o[m], np.asarray(p.point_source_id)[m], + np.asarray(p.gps_time)[m], np.asarray(p.range_m)[m])) + xyz, origin, psid, t, r = (np.concatenate([q[i] for q in parts]) for i in range(5)) + del parts + if not (origin == 2).any(): + check(False, "gap points present") + return + traj = np.genfromtxt(folder / "trajectory.csv", delimiter=",", names=True, usecols=(0, 11)) + node_in_scan = np.zeros(len(traj), np.int64) + for s in np.unique(traj["scan_index"]): + m = np.where(traj["scan_index"] == s)[0] + node_in_scan[m] = np.arange(len(m)) + node = np.searchsorted(traj["timestamp_unix_s"], t, side="left") - 1 + control = (origin == 0) & (node_in_scan[np.maximum(node, 0)] % 20 == 18) + # trajectory.csv has 1 us time resolution, so points within 0.5 us of a pose may land on either side of it + tg = t[origin == 2] + at_or_before = np.searchsorted(traj["timestamp_unix_s"], tg + 1e-6, side="right") - 1 + before = np.searchsorted(traj["timestamp_unix_s"], tg - 1e-6, side="left") - 1 + piece_end = (node_in_scan[at_or_before] % 20 == 19) | (node_in_scan[np.maximum(before, 0)] % 20 == 19) + exact = node_in_scan[np.maximum(node[origin == 2], 0)] % 20 == 19 + check(np.all(piece_end), f"every gap point lies after the last pose of a step 1 piece " + f"({(~exact).sum()} within the CSV's 1 us rounding of a pose time)") + order = np.argsort(psid, kind="stable") + bounds = np.searchsorted(psid[order], np.arange(psid.max() + 2)) + d_gap = np.full(len(xyz), np.nan, np.float32); d_ctl = np.full(len(xyz), np.nan, np.float32) + for s in range(psid.max() + 1): + win = order[bounds[max(s - 1, 0)]:bounds[min(s + 2, len(bounds) - 1)]] + own = order[bounds[s]:bounds[s + 1]] + g = own[origin[own] == 2]; c = own[control[own]] + base = win[(origin[win] == 0)] + if len(g): + d_gap[g] = cKDTree(xyz[base]).query(xyz[g], workers=-1)[0] + if len(c): + d_ctl[c] = cKDTree(xyz[base[~control[base]]]).query(xyz[c], workers=-1)[0] + gm, cm = origin == 2, control + + def row(label, m_g, m_c): + g = d_gap[m_g] * 1000; c = d_ctl[m_c] * 1000 + print(f" {label:>8} {len(g):>10,} {np.median(g):>9.2f} {np.percentile(g, 99):>8.2f} {len(c):>10,} {np.median(c):>9.2f} {np.percentile(c, 99):>8.2f}") + + print(f" {'range':>8} {'gap pts':>10} {'median mm':>9} {'p99 mm':>8} {'control':>10} {'median mm':>9} {'p99 mm':>8}") + for lo, hi in ((1, 3), (3, 5), (5, 10), (10, 20), (20, 100)): + row(f"{lo}-{hi} m", gm & (r >= lo) & (r < hi), cm & (r >= lo) & (r < hi)) + row("all", gm, cm) + med_g, med_c = np.median(d_gap[gm]), np.median(d_ctl[cm]) + check(med_g <= 1.2 * med_c + 0.0005, f"gap points as close to step 1's points as a missing slice of step 1's own " + f"(median {med_g * 1000:.2f} vs {med_c * 1000:.2f} mm)") + + +def check_reference(laz, step2_dir): + print("reference (point_origin = 0 vs step 2 export)") + ref_file = step2_dir / "all_step_2.laz" + n_ref = laspy.open(ref_file).header.point_count + with laspy.open(laz) as f, laspy.open(ref_file) as g: + it_ref = g.chunk_iterator(CHUNK) + pending = None + n = 0 + max_d = np.zeros(3) + intensity_diff = 0 + psid_diff = 0 + gps_scale = None + for p in f.chunk_iterator(CHUNK): + m = np.asarray(p.point_origin) == 0 + if not m.any(): + continue + ours = np.column_stack([p.x[m], p.y[m], p.z[m]]) + ours_i = np.asarray(p.intensity)[m] + ours_t = np.asarray(p.gps_time)[m] + ours_s = np.asarray(p.point_source_id)[m] + k = 0 + while k < len(ours): + if pending is None or len(pending[0]) == 0: + q = next(it_ref) + pending = (np.column_stack([q.x, q.y, q.z]), np.asarray(q.intensity), np.asarray(q.gps_time), + np.asarray(q.point_source_id)) + take = min(len(ours) - k, len(pending[0])) + max_d = np.maximum(max_d, np.abs(ours[k:k + take] - pending[0][:take]).max(axis=0)) + intensity_diff += int((ours_i[k:k + take] != pending[1][:take]).sum()) + psid_diff += int((ours_s[k:k + take] != pending[3][:take]).sum()) + if gps_scale is None: + gps_scale = pending[2][0] / ours_t[k] + pending = tuple(a[take:] for a in pending) + k += take + n += len(ours) + check(n == n_ref, f"point count {n:,} (step 2: {n_ref:,})") + check(max_d.max() <= 0.0002, f"max |dXYZ| {max_d * 1000} mm <= 0.2 mm") + check(intensity_diff == 0, f"intensity identical ({intensity_diff} differ)") + check(psid_diff == 0, f"point_source_id identical ({psid_diff} differ)") + print(f" step 2 gps_time / ours = {gps_scale:.6g} (step 2 writes the session's ns times x 1e9)") + + +def check_trajectory(folder, step2_dir, stats): + print("trajectory") + traj = np.genfromtxt(folder / "trajectory.csv", delimiter=",", names=True) + print(f" {len(traj):,} poses, {traj['timestamp_unix_s'][0]:.6f} .. {traj['timestamp_unix_s'][-1]:.6f} s") + check(np.all(np.diff(traj["timestamp_unix_s"]) > 0), "timestamps strictly increasing") + check(traj["timestamp_unix_s"][0] <= stats["t_min"] + 0.02 and traj["timestamp_unix_s"][-1] >= stats["t_max"] - 0.02, + "trajectory spans the point times") + q = np.column_stack([traj["qw"], traj["qx"], traj["qy"], traj["qz"]]) + check(np.abs(np.linalg.norm(q, axis=1) - 1).max() < 1e-6, "unit quaternions") + check(set(np.unique(traj["scan_index"]).astype(int).tolist()) == stats["psid"], "scan_index matches point_source_id") + if step2_dir: + ref = np.loadtxt(step2_dir / "trajectories.csv", delimiter=",") + check(len(ref) == len(traj), f"pose count {len(traj):,} (step 2: {len(ref):,})") + dt = np.abs(ref[:, 0] / 1e9 - traj["timestamp_unix_s"]).max() + dxyz = np.abs(ref[:, 2:5] - np.column_stack([traj["x"], traj["y"], traj["z"]])).max() + # step 2: qx qy qz qw; q and -q are the same rotation + rq = ref[:, [8, 5, 6, 7]] + dq = np.minimum(np.abs(rq - q).max(axis=1), np.abs(rq + q).max(axis=1)).max() + check(dt < 1e-6, f"timestamps match step 2 (max diff {dt * 1e6:.3f} us)") + check(dxyz <= 0.00005, f"positions match step 2 (max diff {dxyz * 1000:.3f} mm)") + check(dq < 1e-8, f"orientations match step 2 (max quaternion diff {dq:.2e})") + host = np.abs(ref[:, 1] / 1e3 - traj["host_clock_unix_s"]).max() + check(host < 1e-3, f"host clock matches step 2's Unix column / 1000 (max diff {host:.4f} s)") + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("folder", type=Path) + ap.add_argument("--step2", type=Path) + ap.add_argument("--near-min", type=float, default=0.4) + a = ap.parse_args() + laz = a.folder / "point_cloud.laz" + check_fields(laz) + stats = scan(laz) + check(stats["user_data"] and len(stats["user_data"]) == 1, f"user_data (lidar id) values {sorted(stats['user_data'])}") + print(f" point_source_id {min(stats['psid'])} .. {max(stats['psid'])} ({len(stats['psid'])} scans)") + check(1.5e9 < stats["t_min"] < stats["t_max"] < 2.2e9, f"gps_time in Unix seconds ({stats['t_min']:.3f} .. {stats['t_max']:.3f})") + check_range(stats, a.near_min) + check_alignment(stats) + check_gaps(laz, a.folder) + if a.step2: + check_reference(laz, a.step2) + check_trajectory(a.folder, a.step2, stats) + check((a.folder / "README.txt").is_file(), "README.txt present") + print(f"\n{'ALL CHECKS PASSED' if not failures else f'{len(failures)} CHECK(S) FAILED'}") + return 1 if failures else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/apps/easy_gui/src/batch.cpp b/apps/easy_gui/src/batch.cpp new file mode 100644 index 00000000..99849eb3 --- /dev/null +++ b/apps/easy_gui/src/batch.cpp @@ -0,0 +1,802 @@ +#include "batch.h" + +#include "sleep_blocker.h" +#include "version.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; + +namespace easy_gui +{ + namespace + { + constexpr size_t kLogTailLines = 400; + constexpr const char* kInvalidNameCharacters = "/\\:*?\"<>|"; + + std::string timestamp() + { + const std::time_t now = std::time(nullptr); + std::tm tm{}; +#ifdef _WIN32 + localtime_s(&tm, &now); +#else + localtime_r(&now, &tm); +#endif + std::ostringstream out; + out << std::put_time(&tm, "%Y-%m-%d %H:%M:%S"); + return out.str(); + } + + std::string lowerAscii(std::string s) + { + for (char& c : s) + c = static_cast(std::tolower(static_cast(c))); + return s; + } + + bool hasRecordingFiles(const RecordingFiles& files) + { + return !files.laz_files.empty() && !files.csv_files.empty(); + } + + bool ignoredFolder(const std::string& name) + { + return name.empty() || name[0] == '.' || name[0] == '$' || name == "System Volume Information"; + } + + std::string notARecordingReason(const fs::path& folder, const RecordingFiles& files) + { + std::error_code ec; + if (fs::exists(folder / BatchRunner::kSummaryFileName, ec) || fs::exists(folder / "easy_gui.log", ec)) + return "Easy Forest HDMapping results, not a recording."; + std::string reason = files.laz_files.empty() && files.csv_files.empty() + ? "No point cloud files (lidar*.laz) and no IMU files (imu*.csv)." + : files.laz_files.empty() ? "No point cloud files (lidar*.laz)." + : "No IMU files (imu*.csv)."; + for (const auto& entry : fs::directory_iterator(folder, ec)) + if (entry.is_directory(ec) && !ignoredFolder(entry.path().filename().string())) + { + reason += " Recordings are only looked for one folder level down."; + break; + } + return reason; + } + + //! Folder name with the characters outputNameProblem() rejects replaced. + std::string sanitizedName(const std::string& name) + { + std::string out; + for (char c : name) + out += (static_cast(c) < 32 || std::strchr(kInvalidNameCharacters, c)) ? '_' : c; + while (!out.empty() && (out.back() == ' ' || out.back() == '.')) + out.pop_back(); + if (!outputNameProblem(out).empty()) + out = "recording_" + out; + return outputNameProblem(out).empty() ? out : "recording"; + } + + std::string csvField(const std::string& s) + { + if (s.find_first_of(",\"\r\n") == std::string::npos) + return s; + std::string out = "\""; + for (char c : s) + out += c == '"' ? std::string("\"\"") : std::string(1, c); + return out + "\""; + } + + std::string formatSeconds(double s) + { + std::ostringstream out; + out << std::fixed << std::setprecision(0) << s; + return out.str(); + } + + class PipelineJob : public RecordingJob + { + public: + bool start(const PipelineRequest& request, std::string& error) override + { + return pipeline_.start(request, error); + } + bool busy() const override + { + return pipeline_.busy(); + } + PipelineStatus status() const override + { + return pipeline_.status(); + } + void cancel() override + { + pipeline_.cancel(); + } + std::vector logTail() const override + { + return pipeline_.logTail(); + } + + private: + Pipeline pipeline_; + }; + } // namespace + + std::vector discoverRecordings(const fs::path& parent) + { + std::vector found; + std::error_code ec; + if (!fs::is_directory(parent, ec)) + return found; + + const RecordingFiles own = classifyRecordingFiles(parent); + if (hasRecordingFiles(own)) + { + const fs::path normal = parent.lexically_normal(); + const std::string name = normal.filename().empty() ? normal.parent_path().filename().string() : normal.filename().string(); + found.push_back({ parent, name.empty() ? std::string("recording") : name, true, {} }); + } + + std::vector subfolders; + for (const auto& entry : fs::directory_iterator(parent, ec)) + if (entry.is_directory(ec) && !ignoredFolder(entry.path().filename().string())) + subfolders.push_back(entry.path()); + std::sort(subfolders.begin(), subfolders.end()); + for (const auto& folder : subfolders) + { + const RecordingFiles files = classifyRecordingFiles(folder); + DiscoveredFolder d{ folder, folder.filename().string(), hasRecordingFiles(files), {} }; + if (!d.is_recording) + d.skip_reason = notARecordingReason(folder, files); + found.push_back(std::move(d)); + } + return found; + } + + std::string outputNameProblem(const std::string& name) + { + if (name.empty()) + return "Enter a name."; + if (name == "." || name == "..") + return "'" + name + "' cannot be used as a folder name."; + for (char c : name) + if (static_cast(c) < 32 || std::strchr(kInvalidNameCharacters, c)) + return "A folder name cannot contain / \\ : * ? \" < > |."; + if (name.back() == ' ' || name.back() == '.') + return "A folder name cannot end with a space or a dot."; + if (name.size() > 120) + return "The name is too long."; + // Device names Windows reserves, with or without an extension. + const std::string stem = lowerAscii(name.substr(0, name.find('.'))); + static const std::set kReserved = { "con", "prn", "aux", "nul", "com1", "com2", "com3", "com4", + "com5", "com6", "com7", "com8", "com9", "lpt1", "lpt2", "lpt3", + "lpt4", "lpt5", "lpt6", "lpt7", "lpt8", "lpt9" }; + if (kReserved.contains(stem)) + return "'" + name + "' is reserved by Windows."; + return {}; + } + + bool sameOutputName(const std::string& a, const std::string& b) + { + return lowerAscii(a) == lowerAscii(b); + } + + std::string recordingFingerprint(const RecordingFiles& files) + { + uint64_t bytes = 0; + std::error_code ec; + for (size_t i = 0; i < files.pairs; i++) + bytes += fs::file_size(files.laz_files[i], ec) + fs::file_size(files.csv_files[i], ec); + // The first IMU timestamp tells apart recordings of the same size (e.g. two equally long walks). + std::string first_sample; + if (!files.csv_files.empty()) + { + std::ifstream imu(files.csv_files.front()); + std::string line; + while (first_sample.empty() && std::getline(imu, line)) + if (!line.empty() && std::isdigit(static_cast(line[0]))) + first_sample = line.substr(0, line.find_first_of(" ,\t\r")); + } + return std::to_string(files.pairs) + " file pairs, " + std::to_string(bytes) + " bytes, first IMU sample " + + (first_sample.empty() ? "?" : first_sample); + } + + std::vector defaultOutputNames(const std::vector& recordings, const fs::path& output_root) + { + std::vector names; + for (const auto& r : recordings) + { + const std::string base = sanitizedName(r.name); + const std::string fingerprint = recordingFingerprint(classifyRecordingFiles(r.path)); + auto usable = [&](const std::string& candidate) + { + for (const auto& n : names) + if (sameOutputName(n, candidate)) + return false; + if (output_root.empty()) + return true; + const fs::path folder = output_root / candidate; + std::error_code ec; + if (!fs::exists(folder, ec) || fs::is_empty(folder, ec) || holdsUnfinishedOutput(folder)) + return true; + const auto marker = readDoneMarker(folder); + return marker && marker->fingerprint == fingerprint; + }; + std::string candidate = base; + for (int i = 2; !usable(candidate); i++) + candidate = base + "_" + std::to_string(i); + names.push_back(candidate); + } + return names; + } + + bool writeDoneMarker(const fs::path& output_folder, const DoneMarker& m, std::string& error) + { + const fs::path file = OutputLayout(output_folder).done_marker; + std::ofstream out(file, std::ios::binary); + out << "# Easy Forest HDMapping: this folder holds a finished result. Batch mode skips the recording when the\n" + "# same batch is started again; delete this file to process it again.\n" + << "version=" << kEasyForestVersion << "\n" + << "recording=" << m.recording << "\n" + << "fingerprint=" << m.fingerprint << "\n" + << "finished=" << m.finished << "\n" + << "duration_s=" << formatSeconds(m.duration_s) << "\n" + << "raw_points=" << m.raw_points << "\n" + << "exported_points=" << m.exported_points << "\n"; + out.close(); + if (!out) + { + error = "Could not write '" + file.string() + "'."; + return false; + } + return true; + } + + std::optional readDoneMarker(const fs::path& output_folder) + { + std::ifstream in(OutputLayout(output_folder).done_marker, std::ios::binary); + if (!in) + return std::nullopt; + DoneMarker m; + std::string line; + while (std::getline(in, line)) + { + if (!line.empty() && line.back() == '\r') + line.pop_back(); + const auto eq = line.find('='); + if (line.empty() || line[0] == '#' || eq == std::string::npos) + continue; + const std::string key = line.substr(0, eq), value = line.substr(eq + 1); + if (key == "recording") + m.recording = value; + else if (key == "fingerprint") + m.fingerprint = value; + else if (key == "finished") + m.finished = value; + else if (key == "duration_s") + m.duration_s = std::atof(value.c_str()); + else if (key == "raw_points") + m.raw_points = std::strtoull(value.c_str(), nullptr, 10); + else if (key == "exported_points") + m.exported_points = std::strtoull(value.c_str(), nullptr, 10); + } + if (m.fingerprint.empty()) + return std::nullopt; + return m; + } + + BatchDiskNeed batchDiskNeed(const std::vector& recordings) + { + BatchDiskNeed need; + for (const PreflightResult* r : recordings) + { + need.work_bytes = std::max(need.work_bytes, r->estimated_work_bytes); + need.result_bytes += r->estimated_result_bytes; + } + return need; + } + + Finding batchDiskFinding(const BatchDiskNeed& need, bool space_known, uint64_t available) + { + const std::string text = "about " + formatBytes(need.total()) + " (" + formatBytes(need.work_bytes) + + " temporary for the largest recording + " + formatBytes(need.result_bytes) + " of results for all of them)"; + if (!space_known) + return { Severity::Warning, "Could not determine the free disk space. The batch needs " + text + "." }; + if (available < need.total()) + return { Severity::Error, + "Not enough free disk space for the batch: " + text + " needed, " + formatBytes(available) + + " free. Free some space, choose an output folder on another disk, or select fewer recordings." }; + if (available < need.total() + need.total() / 2) + return { Severity::Warning, + "Free disk space is tight for the batch: " + text + " needed, " + formatBytes(available) + " free." }; + return { Severity::Ok, "Disk space for the batch: " + text + " needed, " + formatBytes(available) + " free." }; + } + + std::string batchOutputProblem(const fs::path& output_root, const std::vector& folders) + { + if (output_root.empty()) + return "Choose an output folder."; + for (const auto& f : folders) + if (f.is_recording && isSameOrInside(output_root, f.path)) + return "The output folder is inside the recording '" + f.name + + "'. Choose a folder elsewhere: Easy Forest HDMapping never writes into a recording."; + return {}; + } + + BatchCheck checkBatchRecording(const fs::path& recording, const fs::path& output_folder) + { + BatchCheck check; + check.preflight = runPreflight(recording, output_folder, true); + check.fingerprint = recordingFingerprint(check.preflight.files); + if (auto marker = readDoneMarker(output_folder); marker && marker->fingerprint == check.fingerprint) + check.done = marker; + return check; + } + + std::string firstError(const PreflightResult& result) + { + for (const auto& f : result.findings) + if (f.severity == Severity::Error) + return f.message; + return {}; + } + + std::vector BatchPlan::toProcess() const + { + std::vector out; + for (size_t i = 0; i < checks.size(); i++) + if (request.items[i].include && !checks[i].done) + out.push_back(&checks[i].preflight); + return out; + } + + BatchPlan planBatch(const fs::path& parent, const fs::path& output_root) + { + BatchPlan plan; + plan.found = discoverRecordings(parent); + plan.request.output_root = output_root; + std::vector recordings; + for (const auto& f : plan.found) + (f.is_recording ? recordings : plan.request.skipped).push_back(f); + const auto names = defaultOutputNames(recordings, output_root); + for (size_t i = 0; i < recordings.size(); i++) + { + plan.checks.push_back(checkBatchRecording(recordings[i].path, output_root / names[i])); + const BatchCheck& check = plan.checks.back(); + BatchItem item{ recordings[i].path, recordings[i].name, names[i] }; + item.total_points = check.preflight.total_points; + if (!check.done && !check.preflight.ok()) + { + item.include = false; + item.exclude_reason = "Check failed: " + firstError(check.preflight); + } + plan.request.items.push_back(std::move(item)); + } + return plan; + } + + const char* batchStateToken(BatchItemState state) + { + switch (state) + { + case BatchItemState::Excluded: + return "excluded"; + case BatchItemState::Pending: + return "pending"; + case BatchItemState::Running: + return "running"; + case BatchItemState::Succeeded: + return "succeeded"; + case BatchItemState::Failed: + return "failed"; + case BatchItemState::Cancelled: + return "cancelled"; + case BatchItemState::AlreadyDone: + return "already_done"; + case BatchItemState::NotRun: + return "not_run"; + } + return ""; + } + + double pipelineFraction(const PipelineStatus& s) + { + constexpr double kOdometryShare = 0.9; + if (s.state == PipelineState::Succeeded) + return 1.0; + switch (s.stage) + { + case PipelineStage::Checking: + case PipelineStage::Staging: + return 0.0; + case PipelineStage::Odometry: + return kOdometryShare * std::clamp(s.step1.overallFraction(), 0.0, 1.0); + case PipelineStage::Export: + return kOdometryShare + (1.0 - kOdometryShare) * std::clamp(s.export_fraction, 0.0, 1.0); + case PipelineStage::Cleanup: + return s.session_folder.empty() ? 0.0 : 1.0; + } + return 0.0; + } + + // ------------------------------------------------------------------------------------------------------------ + + BatchRunner::BatchRunner(RecordingJobFactory factory) + : factory_(std::move(factory)) + { + if (!factory_) + factory_ = [] + { + return std::make_unique(); + }; + } + + BatchRunner::~BatchRunner() + { + stopNow(); + wait(); + } + + bool BatchRunner::start(const BatchRequest& request, std::string& error) + { + if (busy_) + { + error = "A batch is already running."; + return false; + } + wait(); + std::error_code ec; + if (!fs::create_directories(request.output_root, ec) && ec) + { + error = "Could not create the output folder '" + request.output_root.string() + "' (" + ec.message() + ")."; + return false; + } + + request_ = request; + request_.settings.clamp(); + { + std::lock_guard lock(mutex_); + status_ = BatchStatus{}; + status_.running = true; + status_.items.resize(request_.items.size()); + std::vector known; + for (const auto& item : request_.items) + if (item.include && item.total_points > 0) + known.push_back(item.total_points); + const double average = + known.empty() ? 1.0 : static_cast(std::accumulate(known.begin(), known.end(), uint64_t{ 0 })) / known.size(); + weights_.assign(request_.items.size(), 0.0); + for (size_t i = 0; i < request_.items.size(); i++) + { + const auto& item = request_.items[i]; + status_.items[i].output_folder = request_.output_root / item.output_name; + status_.items[i].raw_points = item.total_points; + if (!item.include) + { + status_.items[i].state = BatchItemState::Excluded; + status_.items[i].error = item.exclude_reason; + } + else + weights_[i] = item.total_points > 0 ? static_cast(item.total_points) : average; + } + processing_s_ = 0.0; + processed_weight_ = 0.0; + log_tail_.clear(); + started_ = std::chrono::steady_clock::now(); + } + busy_ = true; + thread_ = std::thread(&BatchRunner::run, this); + return true; + } + + void BatchRunner::stopAfterCurrent() + { + std::lock_guard lock(mutex_); + status_.stop_after_current = true; + } + + void BatchRunner::stopNow() + { + std::lock_guard lock(mutex_); + status_.stop_after_current = true; + status_.stop_now = true; + if (job_) + job_->cancel(); + } + + void BatchRunner::wait() + { + if (thread_.joinable()) + thread_.join(); + } + + BatchStatus BatchRunner::status() const + { + std::lock_guard lock(mutex_); + BatchStatus s = status_; + if (s.running) + s.elapsed_s = std::chrono::duration(std::chrono::steady_clock::now() - started_).count(); + return s; + } + + std::vector BatchRunner::logTail() const + { + std::lock_guard lock(mutex_); + return { log_tail_.begin(), log_tail_.end() }; + } + + std::vector BatchRunner::currentLogTail() const + { + std::lock_guard lock(mutex_); + return job_ ? job_->logTail() : std::vector{}; + } + + void BatchRunner::log(const std::string& line) + { + const std::string stamped = "[" + timestamp() + "] " + line; + if (log_file_.is_open()) + log_file_ << stamped << std::endl; // flushed: the log must survive a crash or power loss mid-batch + std::lock_guard lock(mutex_); + log_tail_.push_back(stamped); + while (log_tail_.size() > kLogTailLines) + log_tail_.pop_front(); + } + + void BatchRunner::updateProgress() + { + const double total = std::accumulate(weights_.begin(), weights_.end(), 0.0); + double done = 0.0; + for (size_t i = 0; i < weights_.size(); i++) + { + const BatchItemState st = status_.items[i].state; + if (st != BatchItemState::Pending && st != BatchItemState::Running) + done += weights_[i]; + } + double current_done = 0.0, current_s = 0.0; + if (status_.current >= 0) + { + current_done = weights_[status_.current] * pipelineFraction(status_.current_status); + current_s = std::chrono::duration(std::chrono::steady_clock::now() - item_started_).count(); + } + status_.fraction = total > 0.0 ? std::clamp((done + current_done) / total, 0.0, 1.0) : 1.0; + + // ETA from the processing rate of the recordings that really ran (already-done ones take no time). + double remaining = 0.0; + for (size_t i = 0; i < weights_.size(); i++) + if (status_.items[i].state == BatchItemState::Pending) + remaining += weights_[i]; + if (status_.current >= 0) + remaining += weights_[status_.current] - current_done; + const double processed = processed_weight_ + current_done; + status_.eta_s = (total > 0.0 && processed > 0.02 * total) ? (processing_s_ + current_s) * remaining / processed : -1.0; + } + + void BatchRunner::run() + { + // Several hours of unattended processing: the computer must not go to sleep in between. + SleepBlocker no_sleep("Easy Forest HDMapping is processing recordings"); + + log_file_.open(request_.output_root / kLogFileName, std::ios::out | std::ios::app); + log("batch started: " + versionString()); + log("output folder: " + request_.output_root.string()); + log("lidar odometry program: " + request_.step1_executable.string()); + { + std::ostringstream s; + s << "settings: near_min_range_m=" << request_.settings.near_min_range_m + << " place_by_raw_time=" << request_.settings.place_by_raw_time << " keep_work_files=" << request_.settings.keep_work_files; + log(s.str()); + } + log(std::string("preventing system sleep: ") + (no_sleep.active() ? "yes" : "no (not supported or refused)")); + size_t selected = 0; + for (const auto& item : request_.items) + selected += item.include ? 1 : 0; + log(std::to_string(request_.items.size()) + " recordings found, " + std::to_string(selected) + " selected"); + for (const auto& item : request_.items) + if (!item.include) + log("excluded: " + item.name + (item.exclude_reason.empty() ? "" : " (" + item.exclude_reason + ")")); + for (const auto& folder : request_.skipped) + log("skipped: " + folder.name + " (" + folder.skip_reason + ")"); + writeSummary(); + + for (size_t i = 0; i < request_.items.size(); i++) + { + if (!request_.items[i].include) + continue; + bool stop = false; + { + std::lock_guard lock(mutex_); + stop = status_.stop_after_current; + if (stop) + status_.items[i].state = BatchItemState::NotRun; + } + if (!stop) + processItem(i); + writeSummary(); + } + + BatchStatus final_status = status(); + size_t counts[8] = {}; + for (const auto& r : final_status.items) + counts[static_cast(r.state)]++; + std::ostringstream summary; + summary << "batch " << (final_status.stop_after_current ? "stopped" : "finished") << " after " + << formatSeconds(final_status.elapsed_s) << " s: " << counts[static_cast(BatchItemState::Succeeded)] << " succeeded, " + << counts[static_cast(BatchItemState::Failed)] << " failed, " << counts[static_cast(BatchItemState::AlreadyDone)] + << " already done, " << counts[static_cast(BatchItemState::Cancelled)] << " cancelled, " + << counts[static_cast(BatchItemState::NotRun)] << " not run, " << counts[static_cast(BatchItemState::Excluded)] + << " excluded"; + log(summary.str()); + log_file_.close(); + { + std::lock_guard lock(mutex_); + status_.elapsed_s = std::chrono::duration(std::chrono::steady_clock::now() - started_).count(); + status_.running = false; + status_.current = -1; + updateProgress(); + } + busy_ = false; + } + + void BatchRunner::processItem(size_t index) + { + const BatchItem& item = request_.items[index]; + const fs::path out = request_.output_root / item.output_name; + { + std::lock_guard lock(mutex_); + status_.current = static_cast(index); + status_.current_status = PipelineStatus{}; + status_.items[index].state = BatchItemState::Running; + item_started_ = std::chrono::steady_clock::now(); + } + log("recording " + item.name + ": " + item.recording.string() + " -> " + out.string()); + + auto finishItem = [&](BatchItemState state, const std::string& error, bool ran) + { + std::lock_guard lock(mutex_); + BatchItemResult& r = status_.items[index]; + r.state = state; + r.error = error; + if (ran) + { + r.duration_s = std::chrono::duration(std::chrono::steady_clock::now() - item_started_).count(); + processing_s_ += r.duration_s; + processed_weight_ += weights_[index]; + } + status_.current = -1; + updateProgress(); + }; + + const BatchCheck check = checkBatchRecording(item.recording, out); + { + std::lock_guard lock(mutex_); + status_.items[index].raw_points = check.preflight.total_points; + } + if (check.done) + { + { + std::lock_guard lock(mutex_); + status_.items[index].duration_s = check.done->duration_s; + status_.items[index].exported_points = check.done->exported_points; + } + finishItem(BatchItemState::AlreadyDone, "", false); + log("recording " + item.name + ": already done (finished " + check.done->finished + "), skipped"); + return; + } + if (!check.preflight.ok()) + { + const std::string error = firstError(check.preflight); + finishItem(BatchItemState::Failed, error, false); + log("recording " + item.name + ": FAILED the check: " + error); + return; + } + + std::unique_ptr job = factory_(); + PipelineRequest request{ item.recording, out, request_.step1_executable, request_.settings, request_.check_band_outer_m }; + request.resume_unfinished = true; + std::string error; + if (!job->start(request, error)) + { + finishItem(BatchItemState::Failed, error, true); + log("recording " + item.name + ": FAILED to start: " + error); + return; + } + { + std::lock_guard lock(mutex_); + job_ = job.get(); + if (status_.stop_now) // stopNow() may have run before job_ was published + job_->cancel(); + } + PipelineStage last_stage = PipelineStage::Checking; + while (job->busy()) + { + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + const PipelineStatus s = job->status(); + if (s.stage != last_stage && s.state == PipelineState::Running) + { + static const char* kStageNames[] = { "checking", "preparing", "lidar odometry", "export", "cleaning up" }; + log("recording " + item.name + ": " + kStageNames[static_cast(s.stage)]); + last_stage = s.stage; + } + std::lock_guard lock(mutex_); + status_.current_status = s; + updateProgress(); + } + const PipelineStatus s = job->status(); + { + std::lock_guard lock(mutex_); + job_ = nullptr; + status_.current_status = s; + } + job.reset(); + + if (s.state == PipelineState::Succeeded) + { + const ExportStats& e = s.export_stats; + const uint64_t exported = e.odometry_points + e.near_points + e.gap_points + e.check_band_points; + DoneMarker marker{ + item.recording.string(), check.fingerprint, timestamp(), s.elapsed_s, check.preflight.total_points, exported + }; + std::string marker_error; + const bool marked = writeDoneMarker(out, marker, marker_error); + { + std::lock_guard lock(mutex_); + status_.items[index].exported_points = exported; + } + finishItem(BatchItemState::Succeeded, marked ? "" : marker_error, true); + log("recording " + item.name + ": succeeded in " + formatSeconds(s.elapsed_s) + " s, " + std::to_string(exported) + + " points exported" + (marked ? "" : "; " + marker_error)); + } + else if (s.state == PipelineState::Cancelled) + { + finishItem(BatchItemState::Cancelled, s.message, true); + log("recording " + item.name + ": cancelled"); + } + else + { + const std::string message = s.message.empty() ? "Processing failed." : s.message; + finishItem(BatchItemState::Failed, message, true); + log("recording " + item.name + ": FAILED: " + message + (s.detail.empty() ? "" : " (" + s.detail + ")")); + } + } + + void BatchRunner::writeSummary() + { + const BatchStatus s = status(); + const fs::path file = request_.output_root / kSummaryFileName; + const fs::path temporary = request_.output_root / (std::string(kSummaryFileName) + ".tmp"); + { + std::ofstream out(temporary, std::ios::binary); + out << "recording,output_name,status,duration_s,raw_points,exported_points,output_path,error\n"; + for (size_t i = 0; i < request_.items.size(); i++) + { + const BatchItem& item = request_.items[i]; + const BatchItemResult& r = s.items[i]; + std::error_code ec; + const bool has_output = r.state != BatchItemState::Excluded && fs::exists(r.output_folder, ec); + out << csvField(item.recording.string()) << ',' << csvField(item.output_name) << ',' << batchStateToken(r.state) << ',' + << (r.duration_s > 0.0 ? formatSeconds(r.duration_s) : "") << ',' << (r.raw_points ? std::to_string(r.raw_points) : "") + << ',' << (r.exported_points ? std::to_string(r.exported_points) : "") << ',' + << csvField(has_output ? r.output_folder.string() : "") << ',' << csvField(r.error) << '\n'; + } + for (const auto& folder : request_.skipped) + out << csvField(folder.path.string()) << ",,skipped,,,,," << csvField(folder.skip_reason) << '\n'; + out.close(); + if (!out) + { + log("WARNING: could not write " + temporary.string()); + return; + } + } + std::error_code ec; + fs::rename(temporary, file, ec); + if (ec) // e.g. the summary is open in a spreadsheet program on Windows + log("WARNING: could not update " + file.string() + " (" + ec.message() + ")"); + } +} // namespace easy_gui diff --git a/apps/easy_gui/src/batch.h b/apps/easy_gui/src/batch.h new file mode 100644 index 00000000..bf6b2109 --- /dev/null +++ b/apps/easy_gui/src/batch.h @@ -0,0 +1,265 @@ +#pragma once + +#include "pipeline.h" +#include "preflight.h" +#include "settings.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// Batch mode: several recordings, one after another, through the unchanged single-recording Pipeline. Each recording +// gets its own output folder // with the usual result files; the output root also holds +// batch.log and batch_summary.csv. See docs/DESIGN.md, "Batch mode". +namespace easy_gui +{ + //! A folder found by discoverRecordings(). + struct DiscoveredFolder + { + std::filesystem::path path; + std::string name; // folder name (the parent's own name when the parent itself is a recording) + bool is_recording = false; + std::string skip_reason; // plain language, when !is_recording + }; + + //! The recordings among `parent`'s direct subfolders, plus `parent` itself if it is a recording (listed first). + //! Not recursive: a recording is the folder the scanner wrote, and searching deeper would also find step 1's own + //! folders in old manual runs. A recording is a folder with at least one point cloud and one IMU file as + //! classifyRecordingFiles sees them; the pre-flight check judges the rest. Subfolders that are not recordings are + //! listed with the reason; hidden and system folders (".*", "$*", "System Volume Information") are left out. + std::vector discoverRecordings(const std::filesystem::path& parent); + + //! Why `name` cannot be used as an output folder name, or empty if it can. It must be a single folder name that + //! is valid on both Windows and macOS. + std::string outputNameProblem(const std::string& name); + + //! Output names are compared ignoring (ASCII) case: the default file systems of macOS and Windows do. + bool sameOutputName(const std::string& a, const std::string& b); + + //! Identifies a recording independently of where it is mounted (a USB stick may get another drive letter): + //! number of file pairs, the total size of the paired files and the first IMU timestamp. Reads file sizes and the + //! first lines of one IMU file only. + std::string recordingFingerprint(const RecordingFiles& files); + + //! Default output names: the folder names (characters invalid in a folder name replaced by '_'), made unique + //! ignoring case with _2, _3, ... A name is also passed over when its folder under `output_root` already holds + //! something other than this recording's finished result or an unfinished attempt. + std::vector defaultOutputNames(const std::vector& recordings, const std::filesystem::path& output_root); + + //! Contents of easy_gui_done.txt, the marker batch mode writes into a recording's output folder once its result + //! is complete. A restarted batch skips recordings whose marker matches. + struct DoneMarker + { + std::string recording; // path at the time it was processed (informative) + std::string fingerprint; // recordingFingerprint(): what identifies the recording + std::string finished; // local time + double duration_s = 0.0; + uint64_t raw_points = 0; + uint64_t exported_points = 0; + }; + + bool writeDoneMarker(const std::filesystem::path& output_folder, const DoneMarker& marker, std::string& error); + std::optional readDoneMarker(const std::filesystem::path& output_folder); + + //! Disk space a batch needs at its start: the recordings run one at a time and each frees its temporary files + //! when it ends, so only the largest temporary need counts, plus the results of all of them. + struct BatchDiskNeed + { + uint64_t work_bytes = 0; + uint64_t result_bytes = 0; + uint64_t total() const + { + return work_bytes + result_bytes; + } + }; + + BatchDiskNeed batchDiskNeed(const std::vector& recordings); + + //! The batch disk finding, with the same thresholds as the single-recording check (error below the estimate, + //! warning below 1.5x). `available` is ignored when `space_known` is false. + Finding batchDiskFinding(const BatchDiskNeed& need, bool space_known, uint64_t available); + + //! Why the batch cannot write into `output_root`, or empty: batch.log and batch_summary.csv go there, so it must + //! not be inside any recording (the per-recording check catches the recordings' own output folders). + std::string batchOutputProblem(const std::filesystem::path& output_root, const std::vector& folders); + + //! Pre-flight result of one batch recording, as shown in the table. + struct BatchCheck + { + PreflightResult preflight; // checked with resume_unfinished, against / + std::string fingerprint; + std::optional done; // set if the output folder already holds this recording's finished result + }; + + BatchCheck checkBatchRecording(const std::filesystem::path& recording, const std::filesystem::path& output_folder); + + //! First error of a pre-flight result (empty if none). + std::string firstError(const PreflightResult& result); + + // ------------------------------------------------------------------------------------------------------------ + // Running + + enum class BatchItemState + { + Excluded, // not selected, or the check failed + Pending, + Running, + Succeeded, + Failed, + Cancelled, // stopped by "stop now" while it ran + AlreadyDone, // finished in an earlier run of the same batch + NotRun, // the batch was stopped before it got there + }; + + //! Token in batch_summary.csv ("succeeded", "already_done", ...). + const char* batchStateToken(BatchItemState state); + + struct BatchItem + { + std::filesystem::path recording; + std::string name; // as discovered (shown and logged) + std::string output_name; + bool include = true; + std::string exclude_reason; // written to the summary when !include + uint64_t total_points = 0; // from the check; weights the overall progress (0 = unknown, use the average) + }; + + struct BatchRequest + { + std::filesystem::path output_root; + std::vector items; + std::vector skipped; // folders that are not recordings: only for the summary and the log + std::filesystem::path step1_executable; + Settings settings; + double check_band_outer_m = 0.0; // dev/test option, passed to every recording + }; + + struct BatchItemResult + { + BatchItemState state = BatchItemState::Pending; + double duration_s = 0.0; + uint64_t raw_points = 0; + uint64_t exported_points = 0; + std::string error; + std::filesystem::path output_folder; + }; + + struct BatchStatus + { + bool running = false; + std::vector items; // parallel to BatchRequest::items + int current = -1; // index of the item being processed + PipelineStatus current_status; // of the item being processed + double elapsed_s = 0.0; + double fraction = 0.0; // overall, 0..1, weighted by points + double eta_s = -1.0; // < 0 while unknown + bool stop_after_current = false; + bool stop_now = false; + }; + + //! Share of one recording's processing that is done, 0..1. Odometry is ~90 % of the time, the export the rest + //! (§9 of DESIGN.md: ~21 min odometry, ~2.5 min export). + double pipelineFraction(const PipelineStatus& status); + + //! Processes one recording. The batch runs a Pipeline; tests use a fake. + class RecordingJob + { + public: + virtual ~RecordingJob() = default; + virtual bool start(const PipelineRequest& request, std::string& error) = 0; + virtual bool busy() const = 0; + virtual PipelineStatus status() const = 0; + virtual void cancel() = 0; + virtual std::vector logTail() const + { + return {}; + } + }; + + using RecordingJobFactory = std::function()>; + + //! Runs a batch on a worker thread, one recording after another. A failed recording is marked failed and the + //! batch goes on. Each finished recording gets a done marker; recordings with a matching marker are skipped, so + //! restarting the same batch continues where it stopped. batch_summary.csv is rewritten after every recording, + //! so it is current even if the batch never ends normally. System sleep is prevented while it runs. + //! What happens before "Start", done synchronously (for --batch and tests; the GUI checks row by row in the + //! background): discover, name, check. Recordings that fail the check are not selected (exclude_reason says + //! why); finished ones stay selected, so the summary lists them as already done. + struct BatchPlan + { + std::vector found; // everything discoverRecordings() returned + BatchRequest request; // items + skipped filled; the caller sets step1_executable and the settings + std::vector checks; // parallel to request.items + + //! Pre-flight results of the recordings that will really be processed (selected, not yet done). + std::vector toProcess() const; + }; + + BatchPlan planBatch(const std::filesystem::path& parent, const std::filesystem::path& output_root); + + class BatchRunner + { + public: + //! An empty factory runs the real Pipeline. + explicit BatchRunner(RecordingJobFactory factory = {}); + ~BatchRunner(); // stops now and joins + + bool start(const BatchRequest& request, std::string& error); + //! Lets the running recording finish, then stops. + void stopAfterCurrent(); + //! Cancels the running recording (its unfinished results are removed, as in single mode) and stops. + void stopNow(); + bool busy() const + { + return busy_.load(); + } + //! Blocks until the worker thread has finished. + void wait(); + + BatchStatus status() const; + const BatchRequest& request() const + { + return request_; // only changed by start(), which refuses while busy + } + //! Last lines of batch.log, oldest first. + std::vector logTail() const; + //! Last lines of the running recording's own log. + std::vector currentLogTail() const; + + static constexpr const char* kLogFileName = "batch.log"; + static constexpr const char* kSummaryFileName = "batch_summary.csv"; + + private: + void run(); + void processItem(size_t index); + void log(const std::string& line); + void writeSummary(); + void updateProgress(); // called with mutex_ held + + RecordingJobFactory factory_; + BatchRequest request_; + std::thread thread_; + std::atomic busy_{ false }; + + mutable std::mutex mutex_; + BatchStatus status_; + std::vector weights_; // per item, for the overall fraction + double processing_s_ = 0.0; // time spent in finished recordings that really ran (for the ETA) + double processed_weight_ = 0.0; // their weight + std::chrono::steady_clock::time_point started_; + std::chrono::steady_clock::time_point item_started_; + RecordingJob* job_ = nullptr; // valid while a recording runs; guarded by mutex_ + std::deque log_tail_; + std::ofstream log_file_; // only touched by the worker thread + }; +} // namespace easy_gui diff --git a/apps/easy_gui/src/child_process.cpp b/apps/easy_gui/src/child_process.cpp new file mode 100644 index 00000000..f7efbe53 --- /dev/null +++ b/apps/easy_gui/src/child_process.cpp @@ -0,0 +1,414 @@ +#include "child_process.h" + +#include +#include +#include + +#ifdef _WIN32 +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#else +#include +#include +#include +#include +#include +extern char** environ; +#endif + +#ifdef __APPLE__ +#include +#endif + +namespace fs = std::filesystem; + +namespace easy_gui +{ +#ifdef _WIN32 + struct ChildProcess::Impl + { + HANDLE process = nullptr; + HANDLE job = nullptr; + HANDLE read_pipe = nullptr; + bool waited = false; + int exit_code = -1; + }; + + namespace + { + std::wstring widen(const std::string& s) + { + if (s.empty()) + return {}; + const int n = MultiByteToWideChar(CP_UTF8, 0, s.data(), static_cast(s.size()), nullptr, 0); + std::wstring out(n, L'\0'); + MultiByteToWideChar(CP_UTF8, 0, s.data(), static_cast(s.size()), out.data(), n); + return out; + } + + // Quoting rules of CommandLineToArgvW / the MSVC runtime: backslashes are literal except directly before a + // double quote, where they have to be doubled. + std::wstring quoteArgument(const std::wstring& arg) + { + if (!arg.empty() && arg.find_first_of(L" \t\n\v\"") == std::wstring::npos) + return arg; + std::wstring out = L"\""; + for (auto it = arg.begin();; ++it) + { + size_t backslashes = 0; + while (it != arg.end() && *it == L'\\') + { + ++it; + ++backslashes; + } + if (it == arg.end()) + { + out.append(backslashes * 2, L'\\'); + break; + } + if (*it == L'"') + out.append(backslashes * 2 + 1, L'\\'); + else + out.append(backslashes, L'\\'); + out.push_back(*it); + } + out.push_back(L'"'); + return out; + } + + std::string lastErrorMessage() + { + return "Windows error " + std::to_string(GetLastError()); + } + } // namespace + + ChildProcess::ChildProcess() + : impl_(new Impl) + { + } + + ChildProcess::~ChildProcess() + { + if (impl_->process && !impl_->waited) + { + kill(); + wait(); + } + if (impl_->read_pipe) + CloseHandle(impl_->read_pipe); + if (impl_->process) + CloseHandle(impl_->process); + if (impl_->job) + CloseHandle(impl_->job); + delete impl_; + } + + bool ChildProcess::start(const fs::path& executable, const std::vector& arguments, std::string& error) + { + SECURITY_ATTRIBUTES sa{ sizeof(SECURITY_ATTRIBUTES), nullptr, TRUE }; + HANDLE read_pipe = nullptr; + HANDLE write_pipe = nullptr; + if (!CreatePipe(&read_pipe, &write_pipe, &sa, 0)) + { + error = "Could not create a pipe (" + lastErrorMessage() + ")"; + return false; + } + SetHandleInformation(read_pipe, HANDLE_FLAG_INHERIT, 0); + + HANDLE nul = CreateFileW(L"NUL", GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE, &sa, OPEN_EXISTING, 0, nullptr); + + // The job object makes kill() take down the whole process tree and also kills the child if Easy GUI itself + // dies (JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE). + HANDLE job = CreateJobObjectW(nullptr, nullptr); + if (job) + { + JOBOBJECT_EXTENDED_LIMIT_INFORMATION info{}; + info.BasicLimitInformation.LimitFlags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE; + SetInformationJobObject(job, JobObjectExtendedLimitInformation, &info, sizeof(info)); + } + + std::wstring command_line = quoteArgument(executable.wstring()); + for (const auto& arg : arguments) + command_line += L" " + quoteArgument(widen(arg)); + + STARTUPINFOW si{}; + si.cb = sizeof(si); + si.dwFlags = STARTF_USESTDHANDLES; + si.hStdInput = nul; + si.hStdOutput = write_pipe; + si.hStdError = write_pipe; + + PROCESS_INFORMATION pi{}; + const BOOL ok = CreateProcessW( + executable.wstring().c_str(), + command_line.data(), + nullptr, + nullptr, + TRUE, + CREATE_NO_WINDOW | CREATE_SUSPENDED, + nullptr, + nullptr, + &si, + &pi); + const std::string create_error = ok ? std::string() : lastErrorMessage(); + + CloseHandle(write_pipe); + if (nul != INVALID_HANDLE_VALUE) + CloseHandle(nul); + + if (!ok) + { + CloseHandle(read_pipe); + if (job) + CloseHandle(job); + error = "Could not start '" + executable.string() + "' (" + create_error + ")"; + return false; + } + + if (job) + AssignProcessToJobObject(job, pi.hProcess); + ResumeThread(pi.hThread); + CloseHandle(pi.hThread); + + impl_->process = pi.hProcess; + impl_->job = job; + impl_->read_pipe = read_pipe; + return true; + } + + bool ChildProcess::readLine(std::string& line) + { + for (;;) + { + const auto newline = buffer_.find('\n'); + if (newline != std::string::npos) + { + line = buffer_.substr(0, newline); + buffer_.erase(0, newline + 1); + std::erase(line, '\r'); + return true; + } + if (eof_ || !impl_->read_pipe) + { + if (buffer_.empty()) + return false; + line.swap(buffer_); + buffer_.clear(); + std::erase(line, '\r'); + return true; + } + char chunk[4096]; + DWORD n = 0; + if (!ReadFile(impl_->read_pipe, chunk, sizeof(chunk), &n, nullptr) || n == 0) + eof_ = true; + else + buffer_.append(chunk, n); + } + } + + void ChildProcess::kill() + { + killed_ = true; + if (impl_->job) + TerminateJobObject(impl_->job, 1); + else if (impl_->process) + TerminateProcess(impl_->process, 1); + } + + int ChildProcess::wait() + { + if (!impl_->process) + return -1; + if (!impl_->waited) + { + WaitForSingleObject(impl_->process, INFINITE); + DWORD code = 0; + impl_->exit_code = GetExitCodeProcess(impl_->process, &code) ? static_cast(code) : -1; + impl_->waited = true; + } + return killed_ ? -1 : impl_->exit_code; + } + + fs::path currentExecutableDirectory() + { + std::wstring buffer(32768, L'\0'); + const DWORD n = GetModuleFileNameW(nullptr, buffer.data(), static_cast(buffer.size())); + if (n == 0 || n >= buffer.size()) + return {}; + buffer.resize(n); + return fs::path(buffer).parent_path(); + } + + void attachToParentConsole() + { + if (!AttachConsole(ATTACH_PARENT_PROCESS)) + return; + FILE* stream = nullptr; + freopen_s(&stream, "CONOUT$", "w", stdout); + freopen_s(&stream, "CONOUT$", "w", stderr); + } + +#else // POSIX + + struct ChildProcess::Impl + { + pid_t pid = -1; + int read_fd = -1; + std::atomic reaped{ false }; + int exit_code = -1; + }; + + ChildProcess::ChildProcess() + : impl_(new Impl) + { + } + + ChildProcess::~ChildProcess() + { + if (impl_->pid > 0 && !impl_->reaped) + { + kill(); + wait(); + } + if (impl_->read_fd >= 0) + close(impl_->read_fd); + delete impl_; + } + + bool ChildProcess::start(const fs::path& executable, const std::vector& arguments, std::string& error) + { + int fds[2]; + if (pipe(fds) != 0) + { + error = std::string("Could not create a pipe (") + std::strerror(errno) + ")"; + return false; + } + // Only the child's copy of the write end may stay open, or the parent never sees end-of-output. + fcntl(fds[0], F_SETFD, FD_CLOEXEC); + fcntl(fds[1], F_SETFD, FD_CLOEXEC); + + posix_spawn_file_actions_t actions; + posix_spawn_file_actions_init(&actions); + posix_spawn_file_actions_addopen(&actions, STDIN_FILENO, "/dev/null", O_RDONLY, 0); + posix_spawn_file_actions_adddup2(&actions, fds[1], STDOUT_FILENO); + posix_spawn_file_actions_adddup2(&actions, fds[1], STDERR_FILENO); + + // Own process group, so kill() can signal everything the child starts. + posix_spawnattr_t attributes; + posix_spawnattr_init(&attributes); + posix_spawnattr_setflags(&attributes, POSIX_SPAWN_SETPGROUP); + posix_spawnattr_setpgroup(&attributes, 0); + + const std::string exe = executable.string(); + std::vector argv_storage; + argv_storage.reserve(arguments.size() + 1); + argv_storage.push_back(exe); + argv_storage.insert(argv_storage.end(), arguments.begin(), arguments.end()); + std::vector argv; + for (auto& a : argv_storage) + argv.push_back(a.data()); + argv.push_back(nullptr); + + pid_t pid = -1; + const int rc = posix_spawn(&pid, exe.c_str(), &actions, &attributes, argv.data(), environ); + + posix_spawn_file_actions_destroy(&actions); + posix_spawnattr_destroy(&attributes); + close(fds[1]); + + if (rc != 0) + { + close(fds[0]); + error = "Could not start '" + exe + "' (" + std::strerror(rc) + ")"; + return false; + } + + impl_->pid = pid; + impl_->read_fd = fds[0]; + return true; + } + + bool ChildProcess::readLine(std::string& line) + { + for (;;) + { + const auto newline = buffer_.find('\n'); + if (newline != std::string::npos) + { + line = buffer_.substr(0, newline); + buffer_.erase(0, newline + 1); + std::erase(line, '\r'); + return true; + } + if (eof_ || impl_->read_fd < 0) + { + if (buffer_.empty()) + return false; + line.swap(buffer_); + buffer_.clear(); + std::erase(line, '\r'); + return true; + } + char chunk[4096]; + const ssize_t n = read(impl_->read_fd, chunk, sizeof(chunk)); + if (n > 0) + buffer_.append(chunk, static_cast(n)); + else if (n < 0 && errno == EINTR) + continue; + else + eof_ = true; + } + } + + void ChildProcess::kill() + { + killed_ = true; + // SIGKILL rather than SIGTERM: step 1 installs no handlers, so there is nothing to shut down gracefully, and + // a stuck TBB worker must not be able to delay the cancel. + if (impl_->pid > 0 && !impl_->reaped) + ::kill(-impl_->pid, SIGKILL); + } + + int ChildProcess::wait() + { + if (impl_->pid <= 0) + return -1; + if (!impl_->reaped) + { + int status = 0; + pid_t r; + do + r = waitpid(impl_->pid, &status, 0); + while (r < 0 && errno == EINTR); + impl_->exit_code = (r == impl_->pid && WIFEXITED(status)) ? WEXITSTATUS(status) : -1; + impl_->reaped = true; + } + return impl_->exit_code; + } + + fs::path currentExecutableDirectory() + { +#ifdef __APPLE__ + uint32_t size = 0; + _NSGetExecutablePath(nullptr, &size); + std::string buffer(size, '\0'); + if (_NSGetExecutablePath(buffer.data(), &size) != 0) + return {}; + buffer.resize(std::strlen(buffer.c_str())); + std::error_code ec; + const fs::path resolved = fs::weakly_canonical(buffer, ec); + return (ec ? fs::path(buffer) : resolved).parent_path(); +#else + std::error_code ec; + const fs::path exe = fs::read_symlink("/proc/self/exe", ec); + return ec ? fs::path() : exe.parent_path(); +#endif + } + + void attachToParentConsole() + { + } +#endif +} // namespace easy_gui diff --git a/apps/easy_gui/src/child_process.h b/apps/easy_gui/src/child_process.h new file mode 100644 index 00000000..6340c2e9 --- /dev/null +++ b/apps/easy_gui/src/child_process.h @@ -0,0 +1,63 @@ +#pragma once + +#include +#include +#include +#include +#include + +namespace easy_gui +{ + //! Runs one external program with stdout and stderr merged into a single pipe and stdin closed. + //! + //! Easy GUI runs lidar odometry as a child process instead of a thread because the step 1 sources have + //! abort()/exit() paths (corrupt LAZ, calibration mismatch, out of memory) that would take the whole GUI down, + //! and because compute_step_2 can only be paused, not cancelled. Killing the child is the cancel. + //! + //! stdin is closed on purpose: load_data waits on std::cin.get() before std::exit on a calibration mismatch, and + //! an EOF stdin lets it exit immediately instead of hanging forever. + //! + //! readLine() is meant for one reader thread; kill() may be called from any other thread and makes a blocked + //! readLine() return false once the child's end of the pipe closes. + class ChildProcess + { + public: + ChildProcess(); + ~ChildProcess(); // kills a still-running child and reaps it + ChildProcess(const ChildProcess&) = delete; + ChildProcess& operator=(const ChildProcess&) = delete; + + bool start(const std::filesystem::path& executable, const std::vector& arguments, std::string& error); + + //! Next output line without its line terminator ('\r' characters are removed too: step 1 prints progress + //! as "\rrunning iterations: ..."). Returns false at end of output. + bool readLine(std::string& line); + + //! Terminates the child (on POSIX its whole process group, so helper processes go too). Safe to call twice. + void kill(); + + //! Waits for the child to exit and returns its exit code, or -1 if it was killed by a signal / could not be + //! waited for. + int wait(); + + bool wasKilled() const + { + return killed_.load(); + } + + private: + struct Impl; + Impl* impl_ = nullptr; + std::string buffer_; + bool eof_ = false; + std::atomic killed_{ false }; + }; + + //! Directory of the running executable (empty if it cannot be determined). + std::filesystem::path currentExecutableDirectory(); + + //! Windows: easy_gui.exe is a windowed program, so it has no console and printf goes nowhere. The command-line + //! modes (--headless, --export-only) call this to print into the console they were started from, if any. + //! No-op elsewhere. + void attachToParentConsole(); +} // namespace easy_gui diff --git a/apps/easy_gui/src/easy_gui.manifest b/apps/easy_gui/src/easy_gui.manifest new file mode 100644 index 00000000..21db78cd --- /dev/null +++ b/apps/easy_gui/src/easy_gui.manifest @@ -0,0 +1,11 @@ + + + + + + UTF-8 + + + diff --git a/apps/easy_gui/src/easy_gui_main.cpp b/apps/easy_gui/src/easy_gui_main.cpp new file mode 100644 index 00000000..bacf75f9 --- /dev/null +++ b/apps/easy_gui/src/easy_gui_main.cpp @@ -0,0 +1,1574 @@ +// HDMapping Easy GUI: raw Mandeye recording -> lidar odometry (precision forestry preset) -> point cloud + trajectory. +// See apps/easy_gui/docs/DESIGN.md for the decisions behind the pipeline. + +#include "batch.h" +#include "child_process.h" +#include "export.h" +#include "pipeline.h" +#include "preflight.h" +#include "settings.h" +#include "version.h" + +#include +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; +using namespace easy_gui; + +namespace +{ + const ImVec4 kGreen(0.35f, 0.80f, 0.40f, 1.0f); + const ImVec4 kAmber(0.95f, 0.70f, 0.20f, 1.0f); + const ImVec4 kRed(0.95f, 0.35f, 0.35f, 1.0f); + const ImVec4 kGrey(0.60f, 0.60f, 0.60f, 1.0f); + + fs::path homeFolder() + { +#ifdef _WIN32 + const char* home = std::getenv("USERPROFILE"); +#else + const char* home = std::getenv("HOME"); +#endif + return home ? fs::path(home) : fs::current_path(); + } + + //! ~/Documents/Easy Forest HDMapping/, with _2, _3, ... appended if that already exists. Never next to + //! the recording: that may be the scanner's USB stick. + fs::path defaultOutputFolder(const fs::path& recording) + { + const fs::path base = homeFolder() / "Documents" / "Easy Forest HDMapping"; + const std::string name = recording.filename().empty() ? recording.parent_path().filename().string() : recording.filename().string(); + std::error_code ec; + fs::path candidate = base / name; + for (int i = 2; fs::exists(candidate, ec); i++) + candidate = base / (name + "_" + std::to_string(i)); + return candidate; + } + + std::string formatDuration(double seconds) + { + const int s = static_cast(seconds + 0.5); + char buffer[32]; + if (s >= 3600) + std::snprintf(buffer, sizeof(buffer), "%dh %02dm", s / 3600, (s / 60) % 60); + else + std::snprintf(buffer, sizeof(buffer), "%dm %02ds", s / 60, s % 60); + return buffer; + } + + void wrappedColored(const ImVec4& color, const std::string& text) + { + ImGui::PushStyleColor(ImGuiCol_Text, color); + ImGui::TextWrapped("%s", text.c_str()); + ImGui::PopStyleColor(); + } + + std::string folderName(const fs::path& folder) + { + const fs::path normal = folder.lexically_normal(); + return normal.filename().empty() ? normal.parent_path().filename().string() : normal.filename().string(); + } + + //! ~/Documents/Easy Forest HDMapping/. Unlike single mode, an existing folder from an earlier + //! batch is reused (restarting the same batch continues it); _2, _3, ... only when the folder holds something else. + fs::path defaultBatchOutputFolder(const fs::path& parent) + { + const fs::path base = homeFolder() / "Documents" / "Easy Forest HDMapping"; + const std::string name = folderName(parent); + std::error_code ec; + fs::path candidate = base / name; + for (int i = 2; + fs::exists(candidate, ec) && !fs::is_empty(candidate, ec) && !fs::exists(candidate / BatchRunner::kSummaryFileName, ec); + i++) + candidate = base / (name + "_" + std::to_string(i)); + return candidate; + } + + //! Checks the batch table's rows one at a time on a worker thread: they read the recordings, which may be on a + //! slow USB stick, and the window must stay responsive. Each result carries the generation it was requested for, + //! so results for a parent folder the user has since replaced are dropped. + class RowChecker + { + public: + struct Result + { + size_t row; + int generation; + fs::path output_folder; // the check was made against this output folder + BatchCheck check; + }; + + RowChecker() + { + thread_ = std::thread(&RowChecker::run, this); + } + ~RowChecker() + { + { + std::lock_guard lock(mutex_); + stop_ = true; + } + cv_.notify_all(); + thread_.join(); + } + + void request(size_t row, int generation, const fs::path& recording, const fs::path& output_folder) + { + std::lock_guard lock(mutex_); + std::erase_if( + queue_, + [&](const Job& j) + { + return j.generation != generation || j.row == row; + }); + queue_.push_back({ row, generation, recording, output_folder }); + cv_.notify_all(); + } + + std::vector take() + { + std::lock_guard lock(mutex_); + return std::exchange(results_, {}); + } + + private: + struct Job + { + size_t row; + int generation; + fs::path recording; + fs::path output_folder; + }; + + void run() + { + std::unique_lock lock(mutex_); + while (true) + { + cv_.wait( + lock, + [&] + { + return stop_ || !queue_.empty(); + }); + if (stop_) + return; + const Job job = queue_.front(); + queue_.pop_front(); + lock.unlock(); + BatchCheck check = checkBatchRecording(job.recording, job.output_folder); + lock.lock(); + results_.push_back({ job.row, job.generation, job.output_folder, std::move(check) }); + } + } + + std::mutex mutex_; + std::condition_variable cv_; + std::deque queue_; + std::vector results_; + bool stop_ = false; + std::thread thread_; // last: starts after the members above exist + }; + + struct BatchRow + { + DiscoveredFolder folder; + std::string output_name; + bool name_edited = false; // keep the user's name when the output folder changes + bool selected = true; // the user's choice; a recording that fails the check is never included + std::optional check; + fs::path checked_output; + + bool checkCurrent(const fs::path& output_root) const + { + return check && checked_output == output_root / output_name; + } + bool usable() const + { + return check && (check->done || check->preflight.ok()); + } + bool included() const + { + return selected && usable(); + } + //! Will write into its output folder, as far as is known: included, or selected and not checked yet. + bool active(const fs::path& output_root) const + { + return selected && (!checkCurrent(output_root) || usable()); + } + }; + + struct BatchUi + { + fs::path parent; + fs::path output_root; + bool output_chosen_by_user = false; + std::vector found; + std::vector rows; // the recordings + std::vector skipped; // the other folders + int generation = 0; + RowChecker checker; + + bool disk_known = false; + uint64_t available = 0; + double disk_checked_at = -1e9; + + BatchRunner runner; + bool started = false; // the runner was started; the table shows the status column until "new batch" + std::string start_error; + + void setParent(const fs::path& folder) + { + parent = folder; + found = discoverRecordings(folder); + rows.clear(); + skipped.clear(); + std::vector recordings; + for (const auto& f : found) + (f.is_recording ? recordings : skipped).push_back(f); + if (!output_chosen_by_user) + output_root = defaultBatchOutputFolder(folder); + const auto names = defaultOutputNames(recordings, output_root); + for (size_t i = 0; i < recordings.size(); i++) + rows.push_back({ recordings[i], names[i] }); + generation++; + checkAll(); + } + + void setOutputRoot(const fs::path& folder) + { + output_root = folder; + output_chosen_by_user = true; + std::vector recordings; + for (const auto& r : rows) + recordings.push_back(r.folder); + const auto names = defaultOutputNames(recordings, output_root); + for (size_t i = 0; i < rows.size(); i++) + if (!rows[i].name_edited) + rows[i].output_name = names[i]; + checkAll(); + } + + void checkRow(size_t i) + { + if (outputNameProblem(rows[i].output_name).empty()) + checker.request(i, generation, rows[i].folder.path, output_root / rows[i].output_name); + } + + void checkAll() + { + for (size_t i = 0; i < rows.size(); i++) + checkRow(i); + disk_checked_at = -1e9; + } + + void poll() + { + for (auto& r : checker.take()) + if (r.generation == generation && r.row < rows.size()) + { + rows[r.row].check = std::move(r.check); + rows[r.row].checked_output = r.output_folder; + disk_checked_at = -1e9; + } + // Free space changes while other programs run; refreshed every few seconds before the start. + if (!runner.busy() && !started && ImGui::GetTime() - disk_checked_at > 3.0 && !output_root.empty()) + { + fs::path checked; + disk_known = availableSpace(output_root, available, checked); + disk_checked_at = ImGui::GetTime(); + } + } + + bool checking() const + { + for (const auto& r : rows) + if (!r.checkCurrent(output_root) && outputNameProblem(r.output_name).empty()) + return true; + return false; + } + + //! Why row `i`'s output name cannot be used, or empty. + std::string nameProblem(size_t i) const + { + if (std::string p = outputNameProblem(rows[i].output_name); !p.empty()) + return p; + for (size_t j = 0; j < rows.size(); j++) + if (j != i && rows[j].active(output_root) && sameOutputName(rows[i].output_name, rows[j].output_name)) + return "Another recording uses the same output folder name."; + return {}; + } + + std::vector toProcess() const + { + std::vector out; + for (const auto& r : rows) + if (r.included() && r.checkCurrent(output_root) && !r.check->done) + out.push_back(&r.check->preflight); + return out; + } + + Finding diskFinding() const + { + return batchDiskFinding(batchDiskNeed(toProcess()), disk_known, available); + } + + bool readyToStart() const + { + if (runner.busy() || started || rows.empty() || checking() || !batchOutputProblem(output_root, found).empty()) + return false; + bool any = false; + for (size_t i = 0; i < rows.size(); i++) + { + if (!rows[i].active(output_root)) + continue; + if (!nameProblem(i).empty()) + return false; + any = any || rows[i].included(); + } + return any && diskFinding().severity != Severity::Error; + } + + BatchRequest makeRequest(const Settings& settings) const + { + BatchRequest request; + request.output_root = output_root; + request.skipped = skipped; + request.step1_executable = findStep1Executable(); + request.settings = settings; + for (const auto& r : rows) + { + BatchItem item{ r.folder.path, r.folder.name, r.output_name, r.included() }; + item.total_points = r.check ? r.check->preflight.total_points : 0; + if (!r.selected) + item.exclude_reason = "Not selected."; + else if (!r.usable()) + item.exclude_reason = "Check failed: " + firstError(r.check->preflight); + request.items.push_back(std::move(item)); + } + return request; + } + + void reset() + { + runner.wait(); + parent.clear(); + found.clear(); + rows.clear(); + skipped.clear(); + generation++; + if (!output_chosen_by_user) + output_root.clear(); + started = false; + start_error.clear(); + } + }; + + enum class Mode + { + Single, + Batch + }; + + struct App + { + Mode mode = Mode::Single; + BatchUi batch; + + fs::path recording; + fs::path output_folder; + bool output_chosen_by_user = false; + Settings settings; + + std::future preflight_future; + std::optional preflight; + fs::path preflight_for_recording; + fs::path preflight_for_output; + + Pipeline pipeline; + std::string start_error; + bool show_log = false; + + void setRecording(const fs::path& folder) + { + recording = folder; + if (!output_chosen_by_user) + output_folder = defaultOutputFolder(folder); + startPreflight(); + } + + void startPreflight() + { + if (recording.empty()) + return; + preflight.reset(); + preflight_for_recording = recording; + preflight_for_output = output_folder; + preflight_future = std::async(std::launch::async, runPreflight, recording, output_folder, false); + } + + bool checking() const + { + return preflight_future.valid(); + } + + void poll() + { + if (preflight_future.valid() && preflight_future.wait_for(std::chrono::seconds(0)) == std::future_status::ready) + preflight = preflight_future.get(); + } + + bool readyToStart() const + { + return preflight && preflight->ok() && preflight_for_recording == recording && preflight_for_output == output_folder && + !pipeline.busy(); + } + }; + + void drawAdvancedSettings(Settings& settings) + { + if (ImGui::CollapsingHeader("Advanced settings")) + { + ImGui::Indent(); + float near = static_cast(settings.near_min_range_m); + ImGui::SetNextItemWidth(220); + if (ImGui::SliderFloat( + "Closest points kept [m]", + &near, + static_cast(Settings::kNearMinRangeLowest), + static_cast(Settings::kNearMinRangeHighest), + "%.2f m")) + { + settings.near_min_range_m = near; + settings.clamp(); + } + ImGui::SameLine(); + ImGui::TextDisabled("(?)"); + if (ImGui::IsItemHovered()) + ImGui::SetTooltip( + "Points closer to the scanner than this are left out of the point cloud.\n" + "0.4 m (default) removes the operator's body and the handle.\n" + "Lower values keep more of the close surroundings but may include the operator."); + ImGui::Checkbox("Place all points by their measurement time (experimental)", &settings.place_by_raw_time); + ImGui::SameLine(); + ImGui::TextDisabled("(?)"); + if (ImGui::IsItemHovered()) + ImGui::SetTooltip( + "Off (default): the point cloud uses lidar odometry's own points.\n" + "On: every point is taken from the raw recording and placed at its exact measurement time."); + ImGui::Checkbox("Keep intermediate files (for troubleshooting)", &settings.keep_work_files); + ImGui::Unindent(); + } + } + + void drawInputs(App& app, bool locked) + { + ImGui::BeginDisabled(locked); + + ImGui::SeparatorText("1. Recording"); + ImGui::TextWrapped("The folder the Mandeye scanner wrote (lidar*.laz and imu*.csv files). You can also drag it onto this window."); + if (ImGui::Button("Choose recording...")) + { + const std::string folder = mandeye::fd::SelectFolder("Choose the Mandeye recording folder"); + if (!folder.empty()) + app.setRecording(folder); + } + ImGui::SameLine(); + ImGui::TextUnformatted(app.recording.empty() ? "(none chosen)" : app.recording.string().c_str()); + + ImGui::SeparatorText("2. Output folder"); + ImGui::TextWrapped( + "Results go here. It must be a new or empty folder outside the recording; nothing is ever written into the recording."); + if (ImGui::Button("Choose output folder...")) + { + const std::string folder = mandeye::fd::SelectFolder("Choose where to save the results"); + if (!folder.empty()) + { + app.output_folder = folder; + app.output_chosen_by_user = true; + app.startPreflight(); + } + } + ImGui::SameLine(); + ImGui::TextUnformatted( + app.output_folder.empty() ? "(set automatically once a recording is chosen)" : app.output_folder.string().c_str()); + + drawAdvancedSettings(app.settings); + + ImGui::EndDisabled(); + } + + void drawPreflight(App& app) + { + ImGui::SeparatorText("3. Check"); + if (app.recording.empty()) + { + ImGui::TextDisabled("Choose a recording first."); + return; + } + if (app.checking()) + { + ImGui::TextUnformatted("Checking the recording..."); + return; + } + if (!app.preflight) + { + if (ImGui::Button("Check recording")) + app.startPreflight(); + return; + } + if (app.preflight_for_recording != app.recording || app.preflight_for_output != app.output_folder) + { + if (ImGui::Button("Check again")) + app.startPreflight(); + return; + } + for (const auto& f : app.preflight->findings) + { + const char* tag = f.severity == Severity::Error ? "[X]" : f.severity == Severity::Warning ? "[!]" : "[OK]"; + const ImVec4& color = f.severity == Severity::Error ? kRed : f.severity == Severity::Warning ? kAmber : kGreen; + ImGui::TextColored(color, "%s", tag); + ImGui::SameLine(); + ImGui::TextWrapped("%s", f.message.c_str()); + } + if (!app.preflight->ok() && ImGui::Button("Check again")) + app.startPreflight(); + } + + void stageRow(const char* label, int index, int current, bool failed) + { + if (index < current) + ImGui::TextColored(kGreen, "[done]"); + else if (index == current) + ImGui::TextColored(failed ? kRed : kAmber, failed ? "[failed]" : "[ .. ]"); + else + ImGui::TextColored(kGrey, "[ ]"); + ImGui::SameLine(); + ImGui::TextUnformatted(label); + } + + //! What the running pipeline is doing right now: a label and a progress bar. + void drawStageProgress(const PipelineStatus& s) + { + if (s.stage == PipelineStage::Odometry) + { + const double phase = s.step1.phaseFraction(); + std::string label = phaseLabel(s.step1.phase); + if (s.step1.phase == Step1Phase::Odometry && s.step1.chunks_total > 0) + label += " (" + std::to_string(s.step1.chunks_done) + " / " + std::to_string(s.step1.chunks_total) + ")"; + ImGui::TextUnformatted(label.c_str()); + if (phase >= 0.0) + ImGui::ProgressBar(static_cast(phase), ImVec2(-1, 0)); + else + ImGui::ProgressBar(-1.0f * static_cast(ImGui::GetTime()), ImVec2(-1, 0), "working..."); + } + else if (s.stage == PipelineStage::Export) + { + ImGui::TextUnformatted(s.export_step.empty() ? "Exporting..." : s.export_step.c_str()); + ImGui::ProgressBar(static_cast(s.export_fraction), ImVec2(-1, 0)); + } + else + { + ImGui::TextUnformatted(s.stage == PipelineStage::Cleanup ? "Cleaning up..." : "Preparing..."); + ImGui::ProgressBar(-1.0f * static_cast(ImGui::GetTime()), ImVec2(-1, 0), "working..."); + } + } + + void drawRun(App& app) + { + const PipelineStatus s = app.pipeline.status(); + ImGui::SeparatorText("4. Process"); + + if (s.state == PipelineState::Idle) + { + ImGui::BeginDisabled(!app.readyToStart()); + if (ImGui::Button("Start processing", ImVec2(220, 0))) + { + PipelineRequest request{ app.recording, app.output_folder, findStep1Executable(), app.settings }; + app.start_error.clear(); + app.pipeline.start(request, app.start_error); + } + ImGui::EndDisabled(); + if (!app.start_error.empty()) + wrappedColored(kRed, app.start_error); + return; + } + + // Stage list. Stage indices: 0 check+prepare, 1 odometry, 2 export, 3 = all done. + const bool failed = s.state == PipelineState::Failed; + int current = 0; + if (s.stage == PipelineStage::Odometry) + current = 1; + if (s.stage == PipelineStage::Export || (s.stage == PipelineStage::Cleanup && !s.session_folder.empty())) + current = 2; + if (s.state == PipelineState::Succeeded) + current = 3; + stageRow("Check and prepare the recording", 0, current, failed && current == 0); + stageRow("Lidar odometry (precision forestry settings)", 1, current, failed && current == 1); + stageRow("Export point cloud and trajectory", 2, current, failed && current == 2); + + if (s.state == PipelineState::Running) + { + ImGui::Spacing(); + drawStageProgress(s); + if (s.stage == PipelineStage::Odometry) + { + const double overall = s.step1.overallFraction(); + std::string eta = "Elapsed " + formatDuration(s.elapsed_s); + if (overall > 0.05) + eta += ", about " + formatDuration(s.elapsed_s * (1.0 - overall) / overall) + " left"; + ImGui::TextUnformatted("Overall"); + ImGui::ProgressBar(static_cast(overall), ImVec2(-1, 0), eta.c_str()); + } + else if (s.stage == PipelineStage::Export) + ImGui::TextUnformatted(("Elapsed " + formatDuration(s.elapsed_s)).c_str()); + + ImGui::Spacing(); + if (ImGui::Button("Cancel", ImVec2(120, 0))) + ImGui::OpenPopup("Cancel processing?"); + if (ImGui::BeginPopupModal("Cancel processing?", nullptr, ImGuiWindowFlags_AlwaysAutoResize)) + { + ImGui::TextUnformatted("Stop processing? The work done so far will be discarded."); + if (ImGui::Button("Yes, stop", ImVec2(120, 0))) + { + app.pipeline.cancel(); + ImGui::CloseCurrentPopup(); + } + ImGui::SameLine(); + if (ImGui::Button("No, continue", ImVec2(120, 0))) + ImGui::CloseCurrentPopup(); + ImGui::EndPopup(); + } + return; + } + + ImGui::Spacing(); + if (s.state == PipelineState::Succeeded) + { + wrappedColored(kGreen, s.message + " (took " + formatDuration(s.elapsed_s) + ")"); + const OutputLayout out(app.output_folder); + const ExportStats& e = s.export_stats; + ImGui::BulletText( + "%s: %.1f million points (%.1f million from lidar odometry, %.1f million near and %.1f million gap points " + "added), %.0f MB", + out.point_cloud.filename().string().c_str(), + (e.odometry_points + e.near_points + e.gap_points + e.check_band_points) / 1e6, + e.odometry_points / 1e6, + e.near_points / 1e6, + e.gap_points / 1e6, + e.laz_bytes / 1e6); + ImGui::BulletText("%s: %zu scanner poses", out.trajectory.filename().string().c_str(), e.trajectory_nodes); + ImGui::BulletText("%s describes every field of both files.", out.readme.filename().string().c_str()); + } + else if (s.state == PipelineState::Cancelled) + wrappedColored(kAmber, s.message); + else + { + wrappedColored(kRed, s.message); + if (!s.detail.empty()) + ImGui::TextDisabled("Technical detail: %s", s.detail.c_str()); + ImGui::TextDisabled("The full log is in %s", OutputLayout(app.output_folder).log_file.string().c_str()); + } + + if (ImGui::Button("Open output folder")) + OpenURL(app.output_folder.string().c_str()); + ImGui::SameLine(); + if (ImGui::Button("Process another recording")) + { + app.pipeline.reset(); + app.recording.clear(); + app.output_folder.clear(); + app.output_chosen_by_user = false; + app.preflight.reset(); + } + } + + void drawLog(App& app) + { + ImGui::Spacing(); + if (!ImGui::CollapsingHeader("Details (processing log)")) + return; + const auto lines = app.pipeline.logTail(); + ImGui::BeginChild("log", ImVec2(0, 240), ImGuiChildFlags_Borders, ImGuiWindowFlags_HorizontalScrollbar); + for (const auto& line : lines) + ImGui::TextUnformatted(line.c_str()); + if (ImGui::GetScrollY() >= ImGui::GetScrollMaxY() - 5.0f) // follow the end unless the user scrolled up + ImGui::SetScrollHereY(1.0f); + ImGui::EndChild(); + } + + // --------------------------------------------------------------------------------------------------------------- + // Batch mode ("Several recordings") + + void drawBatchInputs(App& app, bool locked) + { + BatchUi& b = app.batch; + ImGui::BeginDisabled(locked); + + ImGui::SeparatorText("1. Folder with recordings"); + ImGui::TextWrapped( + "A folder that contains several recording folders, for example a copy of the scanner's USB stick. Recordings are looked " + "for one folder level down. You can also drag the folder onto this window."); + if (ImGui::Button("Choose folder...")) + { + const std::string folder = mandeye::fd::SelectFolder("Choose the folder that contains the recordings"); + if (!folder.empty()) + b.setParent(folder); + } + ImGui::SameLine(); + ImGui::TextUnformatted(b.parent.empty() ? "(none chosen)" : b.parent.string().c_str()); + + ImGui::SeparatorText("2. Output folder"); + ImGui::TextWrapped( + "Each recording's results go into a folder of its own in here, named in the table below. Choosing the same output folder " + "again continues an interrupted batch: finished recordings are skipped."); + if (ImGui::Button("Choose output folder...")) + { + const std::string folder = mandeye::fd::SelectFolder("Choose where to save the results"); + if (!folder.empty()) + b.setOutputRoot(folder); + } + ImGui::SameLine(); + ImGui::TextUnformatted(b.output_root.empty() ? "(set automatically once a folder is chosen)" : b.output_root.string().c_str()); + + drawAdvancedSettings(app.settings); + ImGui::EndDisabled(); + } + + void drawCheckCell(const BatchRow& row, bool current) + { + if (!current) + { + ImGui::TextDisabled(outputNameProblem(row.output_name).empty() ? "checking..." : "-"); + return; + } + const BatchCheck& c = *row.check; + if (c.done) + { + ImGui::TextColored(kGreen, "already done"); + if (ImGui::IsItemHovered()) + ImGui::SetTooltip("Finished on %s. It will be skipped.", c.done->finished.c_str()); + return; + } + size_t warnings = 0; + for (const auto& f : c.preflight.findings) + warnings += f.severity == Severity::Warning ? 1 : 0; + if (!c.preflight.ok()) + ImGui::TextColored(kRed, "[X] problem"); + else if (warnings > 0) + ImGui::TextColored(kAmber, "[!] %zu note%s", warnings, warnings > 1 ? "s" : ""); + else + ImGui::TextColored(kGreen, "[OK]"); + if (ImGui::IsItemHovered()) + { + ImGui::BeginTooltip(); + ImGui::PushTextWrapPos(ImGui::GetFontSize() * 40.0f); + for (const auto& f : c.preflight.findings) + { + const char* tag = f.severity == Severity::Error ? "[X]" : f.severity == Severity::Warning ? "[!]" : "[OK]"; + ImGui::TextWrapped("%s %s", tag, f.message.c_str()); + } + ImGui::PopTextWrapPos(); + ImGui::EndTooltip(); + } + } + + void drawStatusCell(const BatchItemResult& r, bool current, const BatchStatus& s) + { + switch (r.state) + { + case BatchItemState::Excluded: + ImGui::TextDisabled("not selected"); + break; + case BatchItemState::Pending: + ImGui::TextDisabled("waiting"); + break; + case BatchItemState::Running: + ImGui::TextColored(kAmber, "running %d%%", current ? static_cast(pipelineFraction(s.current_status) * 100) : 0); + break; + case BatchItemState::Succeeded: + ImGui::TextColored(kGreen, "done (%s)", formatDuration(r.duration_s).c_str()); + break; + case BatchItemState::AlreadyDone: + ImGui::TextColored(kGreen, "already done"); + break; + case BatchItemState::Failed: + ImGui::TextColored(kRed, "failed"); + break; + case BatchItemState::Cancelled: + ImGui::TextColored(kAmber, "stopped"); + break; + case BatchItemState::NotRun: + ImGui::TextDisabled("not run"); + break; + } + if (!r.error.empty() && ImGui::IsItemHovered()) + { + ImGui::BeginTooltip(); + ImGui::PushTextWrapPos(ImGui::GetFontSize() * 40.0f); + ImGui::TextWrapped("%s", r.error.c_str()); + ImGui::PopTextWrapPos(); + ImGui::EndTooltip(); + } + } + + void drawBatchTable(App& app, bool locked) + { + BatchUi& b = app.batch; + ImGui::SeparatorText("3. Recordings"); + if (b.parent.empty()) + { + ImGui::TextDisabled("Choose a folder first."); + return; + } + if (b.rows.empty()) + { + wrappedColored( + kRed, + "No recordings were found in this folder or in the folders directly inside it. A recording folder contains " + "lidar*.laz and imu*.csv files."); + } + + const BatchStatus s = b.runner.status(); + const bool show_status = b.started && s.items.size() == b.rows.size(); + if (!b.rows.empty()) + { + const int columns = show_status ? 6 : 5; + const float row_height = ImGui::GetFrameHeightWithSpacing(); + const float height = row_height * static_cast(std::min(b.rows.size(), 10) + 1) + 6.0f; + if (ImGui::BeginTable( + "recordings", + columns, + ImGuiTableFlags_Borders | ImGuiTableFlags_RowBg | ImGuiTableFlags_ScrollY | ImGuiTableFlags_SizingStretchProp, + ImVec2(0, height))) + { + ImGui::TableSetupScrollFreeze(0, 1); + ImGui::TableSetupColumn("Include", ImGuiTableColumnFlags_WidthFixed); + ImGui::TableSetupColumn("Recording", ImGuiTableColumnFlags_WidthStretch, 3.0f); + ImGui::TableSetupColumn("Length", ImGuiTableColumnFlags_WidthFixed); + ImGui::TableSetupColumn("Check", ImGuiTableColumnFlags_WidthFixed); + ImGui::TableSetupColumn("Output folder name", ImGuiTableColumnFlags_WidthStretch, 3.0f); + if (show_status) + ImGui::TableSetupColumn("Status", ImGuiTableColumnFlags_WidthFixed); + ImGui::TableHeadersRow(); + + for (size_t i = 0; i < b.rows.size(); i++) + { + BatchRow& row = b.rows[i]; + const bool current = row.checkCurrent(b.output_root); + ImGui::PushID(static_cast(i)); + ImGui::TableNextRow(); + + ImGui::TableNextColumn(); + ImGui::BeginDisabled(locked || (current && !row.usable())); + bool include = current ? row.included() : row.selected; + if (ImGui::Checkbox("##include", &include)) + row.selected = include; + ImGui::EndDisabled(); + + ImGui::TableNextColumn(); + ImGui::AlignTextToFramePadding(); + ImGui::TextUnformatted(row.folder.name.c_str()); + if (ImGui::IsItemHovered()) + ImGui::SetTooltip("%s", row.folder.path.string().c_str()); + + ImGui::TableNextColumn(); + if (row.check) + ImGui::TextUnformatted(formatDuration(row.check->preflight.duration_s).c_str()); + else + ImGui::TextDisabled("..."); + + ImGui::TableNextColumn(); + drawCheckCell(row, current); + + ImGui::TableNextColumn(); + const std::string problem = row.active(b.output_root) ? b.nameProblem(i) : std::string(); + ImGui::BeginDisabled(locked); + if (!problem.empty()) + ImGui::PushStyleColor(ImGuiCol_FrameBg, ImVec4(0.45f, 0.12f, 0.12f, 1.0f)); + ImGui::SetNextItemWidth(-FLT_MIN); + ImGui::InputText("##output_name", &row.output_name); + if (ImGui::IsItemDeactivatedAfterEdit()) + { + row.name_edited = true; + b.checkRow(i); + } + if (!problem.empty()) + { + ImGui::PopStyleColor(); + if (ImGui::IsItemHovered()) + ImGui::SetTooltip("%s", problem.c_str()); + } + ImGui::EndDisabled(); + + if (show_status) + { + ImGui::TableNextColumn(); + drawStatusCell(s.items[i], s.current == static_cast(i), s); + } + ImGui::PopID(); + } + ImGui::EndTable(); + } + } + + if (!b.started) + { + // Problems in words, below the table (the table only has room for a mark). + for (size_t i = 0; i < b.rows.size(); i++) + { + const BatchRow& row = b.rows[i]; + if (const std::string problem = row.active(b.output_root) ? b.nameProblem(i) : std::string(); !problem.empty()) + wrappedColored(kRed, row.folder.name + ": output folder name: " + problem); + if (!row.checkCurrent(b.output_root) || row.check->done) + continue; + for (const auto& f : row.check->preflight.findings) + if (f.severity != Severity::Ok) + wrappedColored(f.severity == Severity::Error ? kRed : kAmber, row.folder.name + ": " + f.message); + } + if (const std::string problem = batchOutputProblem(b.output_root, b.found); !problem.empty()) + wrappedColored(kRed, problem); + else if (!b.rows.empty() && !b.checking()) + { + const Finding disk = b.diskFinding(); + wrappedColored( + disk.severity == Severity::Error ? kRed + : disk.severity == Severity::Warning ? kAmber + : kGreen, + disk.message); + } + } + + if (!b.skipped.empty() && + ImGui::TreeNode("skipped", "%zu folder%s skipped (not recordings)", b.skipped.size(), b.skipped.size() > 1 ? "s" : "")) + { + for (const auto& f : b.skipped) + ImGui::BulletText("%s: %s", f.name.c_str(), f.skip_reason.c_str()); + ImGui::TreePop(); + } + } + + void drawBatchRun(App& app) + { + BatchUi& b = app.batch; + ImGui::SeparatorText("4. Process"); + if (!b.started) + { + size_t selected = 0; + for (const auto& r : b.rows) + selected += r.included() ? 1 : 0; + ImGui::BeginDisabled(!b.readyToStart()); + const std::string label = "Start processing " + std::to_string(selected) + " recording" + (selected == 1 ? "" : "s"); + if (ImGui::Button(label.c_str(), ImVec2(260, 0))) + { + b.start_error.clear(); + if (b.runner.start(b.makeRequest(app.settings), b.start_error)) + b.started = true; + } + ImGui::EndDisabled(); + if (!b.start_error.empty()) + wrappedColored(kRed, b.start_error); + return; + } + + const BatchStatus s = b.runner.status(); + const BatchRequest& request = b.runner.request(); + size_t selected = 0, finished = 0; + size_t counts[8] = {}; + for (size_t i = 0; i < s.items.size(); i++) + { + counts[static_cast(s.items[i].state)]++; + if (!request.items[i].include) + continue; + selected++; + if (s.items[i].state != BatchItemState::Pending && s.items[i].state != BatchItemState::Running) + finished++; + } + + if (s.running) + { + std::string overall = std::to_string(finished) + " of " + std::to_string(selected) + " recordings finished, elapsed " + + formatDuration(s.elapsed_s); + if (s.eta_s >= 0.0) + overall += ", about " + formatDuration(s.eta_s) + " left"; + ImGui::TextUnformatted("All recordings"); + ImGui::ProgressBar(static_cast(s.fraction), ImVec2(-1, 0), overall.c_str()); + ImGui::Spacing(); + if (s.current >= 0) + { + ImGui::Text( + "Now: %s (%d%% of this recording)", + request.items[s.current].name.c_str(), + static_cast(pipelineFraction(s.current_status) * 100)); + drawStageProgress(s.current_status); + } + else + ImGui::TextUnformatted("Checking the next recording..."); + + ImGui::Spacing(); + ImGui::BeginDisabled(s.stop_after_current); + if (ImGui::Button(s.stop_after_current ? "Stopping after this recording..." : "Stop after this recording", ImVec2(260, 0))) + b.runner.stopAfterCurrent(); + ImGui::EndDisabled(); + ImGui::SameLine(); + if (ImGui::Button("Stop now", ImVec2(120, 0))) + ImGui::OpenPopup("Stop now?"); + if (ImGui::BeginPopupModal("Stop now?", nullptr, ImGuiWindowFlags_AlwaysAutoResize)) + { + ImGui::TextUnformatted( + "Stop now? The recording being processed is discarded.\nRecordings that are already finished are kept."); + if (ImGui::Button("Yes, stop now", ImVec2(140, 0))) + { + b.runner.stopNow(); + ImGui::CloseCurrentPopup(); + } + ImGui::SameLine(); + if (ImGui::Button("No, continue", ImVec2(140, 0))) + ImGui::CloseCurrentPopup(); + ImGui::EndPopup(); + } + ImGui::TextDisabled("The computer is kept from going to sleep while the batch runs."); + return; + } + + const size_t failed = counts[static_cast(BatchItemState::Failed)] + counts[static_cast(BatchItemState::Cancelled)]; + const size_t not_run = counts[static_cast(BatchItemState::NotRun)]; + std::string summary = std::string(s.stop_after_current ? "Batch stopped" : "Batch finished") + " after " + + formatDuration(s.elapsed_s) + ": " + std::to_string(counts[static_cast(BatchItemState::Succeeded)]) + " succeeded"; + if (counts[static_cast(BatchItemState::AlreadyDone)]) + summary += ", " + std::to_string(counts[static_cast(BatchItemState::AlreadyDone)]) + " already done before"; + if (failed) + summary += ", " + std::to_string(failed) + " failed or stopped"; + if (not_run) + summary += ", " + std::to_string(not_run) + " not run"; + summary += "."; + wrappedColored(failed || not_run ? kAmber : kGreen, summary); + if (failed) + ImGui::TextWrapped( + "Point at 'failed' in the table to see why. Each recording's own easy_gui.log has the details. Start the same batch " + "again to retry the recordings that did not finish."); + ImGui::TextDisabled("batch_summary.csv and batch.log are in %s", request.output_root.string().c_str()); + + if (ImGui::Button("Open output folder")) + OpenURL(request.output_root.string().c_str()); + ImGui::SameLine(); + if (ImGui::Button("Process another batch")) + b.reset(); + } + + void drawLogLines(const char* id, const std::vector& lines) + { + ImGui::BeginChild(id, ImVec2(0, 200), ImGuiChildFlags_Borders, ImGuiWindowFlags_HorizontalScrollbar); + for (const auto& line : lines) + ImGui::TextUnformatted(line.c_str()); + if (ImGui::GetScrollY() >= ImGui::GetScrollMaxY() - 5.0f) // follow the end unless the user scrolled up + ImGui::SetScrollHereY(1.0f); + ImGui::EndChild(); + } + + void drawBatchLog(App& app) + { + ImGui::Spacing(); + if (!ImGui::CollapsingHeader("Details (processing log)")) + return; + ImGui::TextUnformatted("Batch (batch.log)"); + drawLogLines("batch_log", app.batch.runner.logTail()); + ImGui::TextUnformatted("Current recording (its easy_gui.log)"); + drawLogLines("recording_log", app.batch.runner.currentLogTail()); + } + + void drawModeSwitch(App& app, bool locked) + { + ImGui::BeginDisabled(locked); + int mode = static_cast(app.mode); + ImGui::RadioButton("One recording", &mode, static_cast(Mode::Single)); + ImGui::SameLine(); + ImGui::RadioButton("Several recordings", &mode, static_cast(Mode::Batch)); + app.mode = static_cast(mode); + ImGui::EndDisabled(); + ImGui::Spacing(); + } +} // namespace + +namespace +{ + //! `easy_gui --headless `: the same pipeline without a window, for scripted runs + //! (the dev validation script) and remote machines. Exit code 0 = success, 1 = failed / refused, 2 = usage. + int runHeadless(const fs::path& recording, const fs::path& output_folder, const Settings& settings, double check_band_outer_m) + { + // Unbuffered, so progress shows up as it happens even when redirected to a file or pipe (MSVC treats _IOLBF as + // full buffering); the output is a line every few seconds. + std::setvbuf(stdout, nullptr, _IONBF, 0); + const PreflightResult check = runPreflight(recording, output_folder); + for (const auto& f : check.findings) + std::printf( + "%s %s\n", + f.severity == Severity::Error ? "[X]" + : f.severity == Severity::Warning ? "[!]" + : "[OK]", + f.message.c_str()); + if (!check.ok()) + return 1; + + Pipeline pipeline; + std::string error; + if (!pipeline.start(PipelineRequest{ recording, output_folder, findStep1Executable(), settings, check_band_outer_m }, error)) + { + std::printf("[X] %s\n", error.c_str()); + return 1; + } + std::string last; + while (pipeline.busy()) + { + const PipelineStatus s = pipeline.status(); + char line[256]; + const double f = s.step1.phaseFraction(); + if (s.stage == PipelineStage::Export) + std::snprintf(line, sizeof(line), "Export: %.0f%%", s.export_fraction * 100.0); + else + std::snprintf( + line, + sizeof(line), + "%s: %s overall %.0f%%", + phaseLabel(s.step1.phase), + f >= 0.0 ? (std::to_string(static_cast(f * 100)) + "%,").c_str() : "", + s.step1.overallFraction() * 100.0); + if (last != line) + std::printf("[%s] %s\n", formatDuration(s.elapsed_s).c_str(), line); + last = line; + std::this_thread::sleep_for(std::chrono::seconds(5)); + } + const PipelineStatus s = pipeline.status(); + std::printf( + "%s %s (%s)\n", s.state == PipelineState::Succeeded ? "[OK]" : "[X]", s.message.c_str(), formatDuration(s.elapsed_s).c_str()); + if (!s.detail.empty()) + std::printf(" detail: %s\n", s.detail.c_str()); + return s.state == PipelineState::Succeeded ? 0 : 1; + } + + std::atomic g_interrupts{ 0 }; + + extern "C" void onInterrupt(int) + { + g_interrupts++; + std::signal(SIGINT, onInterrupt); // some C libraries reset the handler after each signal + } + + std::string pointsText(uint64_t points) + { + char buffer[32]; + std::snprintf(buffer, sizeof(buffer), "%.1f M", points / 1e6); + return buffer; + } + + //! `easy_gui --batch `: batch mode without a window, with the GUI's rules: every + //! recording that passes the check is processed, finished ones are skipped. Ctrl+C once stops after the current + //! recording, twice stops now. Exit code 0 = every selected recording succeeded (or was already done), 1 = some + //! failed, were not run or the batch was refused, 2 = usage. + int runBatchHeadless(const fs::path& parent, const fs::path& output_root, const Settings& settings, double check_band_outer_m) + { + std::setvbuf(stdout, nullptr, _IONBF, 0); + std::error_code ec; + if (!fs::is_directory(parent, ec)) + { + std::printf("[X] The folder '%s' does not exist or is not a folder.\n", parent.string().c_str()); + return 1; + } + std::printf("Looking for recordings in '%s' and checking them...\n", parent.string().c_str()); + BatchPlan plan = planBatch(parent, output_root); + BatchRequest& request = plan.request; + request.step1_executable = findStep1Executable(); + request.settings = settings; + request.check_band_outer_m = check_band_outer_m; + for (const auto& f : request.skipped) + std::printf("[skipped] %s: %s\n", f.name.c_str(), f.skip_reason.c_str()); + if (request.items.empty()) + { + std::printf("[X] No recordings found in '%s' or its subfolders.\n", parent.string().c_str()); + return 1; + } + if (const std::string problem = batchOutputProblem(output_root, plan.found); !problem.empty()) + { + std::printf("[X] %s\n", problem.c_str()); + return 1; + } + + for (size_t i = 0; i < request.items.size(); i++) + { + const BatchItem& item = request.items[i]; + const BatchCheck& c = plan.checks[i]; + const std::string label = item.name + (item.output_name == item.name ? "" : " -> " + item.output_name); + if (c.done) + std::printf("[done] %s: already processed (finished %s)\n", label.c_str(), c.done->finished.c_str()); + else if (!item.include) + std::printf("[X] %s: %s\n", label.c_str(), firstError(c.preflight).c_str()); + else + { + std::printf( + "[OK] %s: %s, %s points\n", + label.c_str(), + formatDuration(c.preflight.duration_s).c_str(), + pointsText(c.preflight.total_points).c_str()); + for (const auto& f : c.preflight.findings) + if (f.severity == Severity::Warning) + std::printf(" [!] %s\n", f.message.c_str()); + } + } + if (std::none_of( + request.items.begin(), + request.items.end(), + [](const BatchItem& item) + { + return item.include; + })) + { + std::printf("[X] None of the recordings passed the check; nothing to process.\n"); + return 1; + } + + uint64_t available = 0; + fs::path checked; + const bool known = availableSpace(output_root, available, checked); + const Finding disk = batchDiskFinding(batchDiskNeed(plan.toProcess()), known, available); + std::printf( + "%s %s\n", + disk.severity == Severity::Error ? "[X]" + : disk.severity == Severity::Warning ? "[!]" + : "[OK]", + disk.message.c_str()); + if (disk.severity == Severity::Error) + return 1; + + BatchRunner runner; + std::string error; + if (!runner.start(request, error)) + { + std::printf("[X] %s\n", error.c_str()); + return 1; + } + std::signal(SIGINT, onInterrupt); + int interrupts_seen = 0; + std::string last; + auto last_print = std::chrono::steady_clock::now() - std::chrono::seconds(5); + while (runner.busy()) + { + std::this_thread::sleep_for(std::chrono::milliseconds(500)); + if (const int n = g_interrupts.load(); n != interrupts_seen) + { + interrupts_seen = n; + if (n == 1) + { + runner.stopAfterCurrent(); + std::printf("Stopping after the current recording. Press Ctrl+C again to stop now.\n"); + } + else + { + runner.stopNow(); + std::printf("Stopping now.\n"); + } + } + const BatchStatus s = runner.status(); + std::string line = "overall " + std::to_string(static_cast(s.fraction * 100)) + "%"; + if (s.eta_s >= 0.0) + line += ", about " + formatDuration(s.eta_s) + " left"; + if (s.current >= 0) + { + const PipelineStatus& p = s.current_status; + std::string stage = p.stage == PipelineStage::Odometry ? phaseLabel(p.step1.phase) + : p.stage == PipelineStage::Export ? "Export" + : "Preparing"; + line += " | " + std::to_string(s.current + 1) + "/" + std::to_string(request.items.size()) + " " + + request.items[s.current].name + ": " + stage + " " + std::to_string(static_cast(pipelineFraction(p) * 100)) + "%"; + } + const auto now = std::chrono::steady_clock::now(); + if (line != last && now - last_print >= std::chrono::seconds(5)) + { + std::printf("[%s] %s\n", formatDuration(s.elapsed_s).c_str(), line.c_str()); + last = line; + last_print = now; + } + } + std::signal(SIGINT, SIG_DFL); + + const BatchStatus s = runner.status(); + bool all_good = true; + for (size_t i = 0; i < request.items.size(); i++) + { + const BatchItemResult& r = s.items[i]; + const bool good = r.state == BatchItemState::Succeeded || r.state == BatchItemState::AlreadyDone; + if (request.items[i].include && !good) + all_good = false; + std::printf( + "%s %s: %s%s%s\n", + good ? "[OK]" + : r.state == BatchItemState::Excluded ? "[--]" + : "[X]", + request.items[i].name.c_str(), + batchStateToken(r.state), + r.error.empty() ? "" : ": ", + r.error.c_str()); + } + std::printf( + "Batch %s after %s. Summary: %s\n", + s.stop_after_current ? "stopped" : "finished", + formatDuration(s.elapsed_s).c_str(), + (output_root / BatchRunner::kSummaryFileName).string().c_str()); + return all_good ? 0 : 1; + } + + //! Dev aid: `easy_gui --export-only [--check-band ]` runs only the + //! export, on an existing step 1 session of that recording (step 1 takes ~20 min on a 10-min walk; the export + //! about a minute). The output folder may exist; the three result files in it are overwritten. + int runExportOnly( + const fs::path& recording, + const fs::path& session_file, + const fs::path& output_folder, + const Settings& settings, + double check_band_outer_m) + { + std::setvbuf(stdout, nullptr, _IONBF, 0); + const PreflightResult check = runPreflight(recording, output_folder); + if (check.files.pairs == 0) + { + std::printf("[X] no lidar/IMU file pairs in '%s'\n", recording.string().c_str()); + return 1; + } + const OutputLayout out(output_folder); + ExportRequest request; + request.session_file = session_file; + request.recording = check.files; + request.lidar_id = std::max(0, check.lidar_id); + request.near_min_range_m = settings.near_min_range_m; + request.check_band_outer_m = check_band_outer_m; + request.place_by_raw_time = settings.place_by_raw_time; + request.laz_file = out.point_cloud; + request.trajectory_file = out.trajectory; + request.readme_file = out.readme; + + std::atomic cancel{ false }; + int last_percent = -1; + ExportStats stats; + std::string error; + const bool ok = exportSession( + request, + cancel, + [&](double fraction, const std::string& step) + { + const int percent = static_cast(fraction * 100); + if (percent / 10 != last_percent / 10) + std::printf("[%3d%%] %s\n", percent, step.c_str()); + last_percent = percent; + }, + stats, + error); + if (!ok) + { + std::printf("[X] %s\n", error.c_str()); + return 1; + } + std::printf( + "[OK] %llu odometry + %llu near + %llu gap + %llu check-band points; raw read %llu, no-return %llu, outside trajectory %llu; " + "range %.3f - %.3f m; %zu poses; LAZ %llu bytes, CSV %llu bytes\n", + static_cast(stats.odometry_points), + static_cast(stats.near_points), + static_cast(stats.gap_points), + static_cast(stats.check_band_points), + static_cast(stats.raw_points_read), + static_cast(stats.raw_no_return), + static_cast(stats.raw_outside_trajectory), + stats.range_min_m, + stats.range_max_m, + stats.trajectory_nodes, + static_cast(stats.laz_bytes), + static_cast(stats.trajectory_bytes)); + return 0; + } + + //! Removes `--check-band ` from the arguments; returns the value, or 0 when absent. + double takeCheckBand(std::vector& args) + { + for (size_t i = 0; i + 1 < args.size(); i++) + if (args[i] == "--check-band") + { + const double value = std::atof(args[i + 1].c_str()); + args.erase(args.begin() + static_cast(i), args.begin() + static_cast(i) + 2); + return value; + } + return 0.0; + } + + //! Removes `flag` from the arguments; returns whether it was there. + bool takeFlag(std::vector& args, const std::string& flag) + { + const auto it = std::find(args.begin(), args.end(), flag); + if (it == args.end()) + return false; + args.erase(it); + return true; + } +} // namespace + +int main(int argc, char* argv[]) +{ + std::vector args(argv + 1, argv + argc); + if (!args.empty() && (args[0] == "--headless" || args[0] == "--export-only" || args[0] == "--batch")) + { + attachToParentConsole(); + const double check_band = takeCheckBand(args); + Settings settings; + settings.place_by_raw_time = takeFlag(args, "--place-by-raw-time"); + if (args[0] == "--headless" && args.size() == 3) + return runHeadless(args[1], args[2], settings, check_band); + if (args[0] == "--export-only" && args.size() == 4) + return runExportOnly(args[1], args[2], args[3], settings, check_band); + if (args[0] == "--batch" && args.size() == 3) + return runBatchHeadless(args[1], args[2], settings, check_band); + std::printf( + "usage: %s --headless [--check-band ] [--place-by-raw-time]\n" + " %s --batch [--check-band ] [--place-by-raw-time]\n" + " %s --export-only [--check-band ] [--place-by-raw-time]\n" + "--batch: processes every recording in (one folder level down; the parent itself too, if it is a\n" + " recording) into //. Recordings finished in an earlier run are skipped.\n" + " Ctrl+C stops after the current recording, a second Ctrl+C stops now.\n" + "--check-band (testing): also export the raw points between the 1.0 m load filter and , flagged\n" + " point_origin = 3, to check that near points line up with the odometry points.\n" + "--place-by-raw-time: place all points from the raw files at their real time (export option, default off).\n", + argv[0], + argv[0], + argv[0]); + return 2; + } + + // Dev aid: `--screenshot [recording | --batch ]` renders ~2 s, saves the window and exits (layout checks + // without capturing the whole desktop). + std::string screenshot_file; + int first_positional = 1; + if (argc > 2 && std::string(argv[1]) == "--screenshot") + { + screenshot_file = argv[2]; + first_positional = 3; + } + + App app; + if (argc > first_positional + 1 && std::string(argv[first_positional]) == "--batch" && fs::is_directory(argv[first_positional + 1])) + { + app.mode = Mode::Batch; + app.batch.setParent(argv[first_positional + 1]); + } + else if (argc > first_positional && fs::is_directory(argv[first_positional])) + app.setRecording(argv[first_positional]); + + SetConfigFlags(FLAG_WINDOW_RESIZABLE | FLAG_MSAA_4X_HINT); + InitWindow(900, 780, versionString().c_str()); + raylib_widgets::fitWindowToScreen(); + // raylib's default exit key (Esc) would close the window mid-run. + SetExitKey(KEY_NULL); + SetTargetFPS(30); + rlImGuiSetup(true); + + int frame = 0; + while (!WindowShouldClose()) + { + app.poll(); + app.batch.poll(); + const bool single_locked = app.pipeline.busy() || app.pipeline.status().state != PipelineState::Idle; + const bool batch_locked = app.batch.runner.busy() || app.batch.started; + const bool locked = app.mode == Mode::Single ? single_locked : batch_locked; + + if (IsFileDropped()) + { + FilePathList dropped = LoadDroppedFiles(); + if (dropped.count > 0 && !locked) + { + fs::path p(dropped.paths[0]); + std::error_code ec; + const fs::path folder = fs::is_directory(p, ec) ? p : p.parent_path(); + if (app.mode == Mode::Single) + app.setRecording(folder); + else + app.batch.setParent(folder); + } + UnloadDroppedFiles(dropped); + } + + BeginDrawing(); + ClearBackground(Color{ 30, 30, 30, 255 }); + rlImGuiBegin(); + + const ImGuiViewport* viewport = ImGui::GetMainViewport(); + ImGui::SetNextWindowPos(viewport->WorkPos); + ImGui::SetNextWindowSize(viewport->WorkSize); + ImGui::Begin( + "Easy Forest HDMapping", + nullptr, + // A scrollbar appears when the content is taller than the window (batch table, open log). + ImGuiWindowFlags_NoTitleBar | ImGuiWindowFlags_NoResize | ImGuiWindowFlags_NoCollapse | ImGuiWindowFlags_NoMove | + ImGuiWindowFlags_NoSavedSettings); + + ImGui::TextUnformatted("Easy Forest HDMapping"); + ImGui::TextDisabled("Mandeye recording -> lidar odometry (precision forestry settings) -> point cloud and trajectory"); + ImGui::Spacing(); + drawModeSwitch(app, single_locked || batch_locked); + + if (app.mode == Mode::Single) + { + drawInputs(app, locked); + drawPreflight(app); + drawRun(app); + drawLog(app); + } + else + { + drawBatchInputs(app, locked); + drawBatchTable(app, locked); + drawBatchRun(app); + drawBatchLog(app); + } + + ImGui::End(); + rlImGuiEnd(); + EndDrawing(); + + // Dev aid for documentation screenshots of a running batch: EASY_GUI_SCREENSHOT_START=1 starts it once ready, + // EASY_GUI_SCREENSHOT_FRAME= takes the screenshot at frame n (30 frames per second) instead of 60. + static const bool auto_start = !screenshot_file.empty() && std::getenv("EASY_GUI_SCREENSHOT_START"); + if (auto_start && app.mode == Mode::Batch && app.batch.readyToStart()) + { + if (app.batch.runner.start(app.batch.makeRequest(app.settings), app.batch.start_error)) + app.batch.started = true; + } + static const int screenshot_frame = + std::getenv("EASY_GUI_SCREENSHOT_FRAME") ? std::max(1, std::atoi(std::getenv("EASY_GUI_SCREENSHOT_FRAME"))) : 60; + if (!screenshot_file.empty() && ++frame == screenshot_frame) + { + TakeScreenshot(screenshot_file.c_str()); + break; + } + } + + // Closing the window while processing stops the child process (Pipeline's destructor cancels and joins). + app.pipeline.cancel(); + app.batch.runner.stopNow(); + rlImGuiShutdown(); + CloseWindow(); + return 0; +} diff --git a/apps/easy_gui/src/export.cpp b/apps/easy_gui/src/export.cpp new file mode 100644 index 00000000..df0a25c1 --- /dev/null +++ b/apps/easy_gui/src/export.cpp @@ -0,0 +1,600 @@ +#include "export.h" + +#include "las14_writer.h" + +#include +#include + +#include "version.h" +#include +#include + +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; + +namespace easy_gui +{ + namespace + { + constexpr double kCoordinateScale = 0.0001; // as step 2's export (LazWriter), so XYZ quantise identically + //! A raw point belongs to the last trajectory node before it, if that node is at most this much older. The same + //! tolerance step 1 applies when it assigns points to nodes (compute_step_1). + constexpr double kNodeTolerance_s = 0.01; + constexpr double kRangeBin_m = 0.1; + + //! Extra bytes of every exported point, in this order. + const std::vector kAttributes = { + { ExtraBytesAttribute::Type::F32, "range_m", "3D distance from the lidar [m]" }, + { ExtraBytesAttribute::Type::U8, "point_origin", "0 step 1, 1 near, 2 gap point" }, + }; + +#pragma pack(push, 1) + struct ExtraBytes + { + float range_m; + uint8_t point_origin; + }; +#pragma pack(pop) + static_assert(sizeof(ExtraBytes) == 5); + + //! Streaming laszip reader: one point at a time, positions with the header's scale and offset applied. + class LazReader + { + public: + ~LazReader() + { + if (reader_) + { + laszip_close_reader(reader_); + laszip_destroy(reader_); + } + } + + bool open(const fs::path& file, std::string& error) + { + laszip_BOOL compressed = 0; + if (laszip_create(&reader_) || laszip_open_reader(reader_, file.string().c_str(), &compressed) || + laszip_get_header_pointer(reader_, &header_) || laszip_get_point_pointer(reader_, &point_)) + { + error = "could not open '" + file.string() + "'"; + return false; + } + count_ = header_->number_of_point_records ? header_->number_of_point_records : header_->extended_number_of_point_records; + return true; + } + + uint64_t count() const + { + return count_; + } + + bool next(std::string& error) + { + if (laszip_read_point(reader_)) + { + error = "could not read a point"; + return false; + } + return true; + } + + Eigen::Vector3d position() const + { + return { header_->x_offset + header_->x_scale_factor * static_cast(point_->X), + header_->y_offset + header_->y_scale_factor * static_cast(point_->Y), + header_->z_offset + header_->z_scale_factor * static_cast(point_->Z) }; + } + + const laszip_point& point() const + { + return *point_; + } + + private: + laszip_POINTER reader_ = nullptr; + laszip_header* header_ = nullptr; + laszip_point* point_ = nullptr; + uint64_t count_ = 0; + }; + + //! One trajectory node with everything the export needs. + struct Node + { + double time_s; // lidar clock (Unix epoch seconds on Mandeye) + size_t scan; + Eigen::Affine3d global; // scan pose (session_poses.mrp) x node pose relative to the scan + //! Last node of a step 1 piece (worker chunk): step 1 takes the points in [first node, last node) of each + //! piece, so raw points from this node until the next piece's first node are not in its output. + bool piece_end; + }; + + void addRange(ExportStats& stats, double range) + { + const uint64_t total = stats.odometry_points + stats.near_points + stats.check_band_points; + if (total == 0 || range < stats.range_min_m) + stats.range_min_m = range; + if (total == 0 || range > stats.range_max_m) + stats.range_max_m = range; + const size_t bin = std::min(stats.range_histogram.size() - 1, static_cast(std::max(0.0, range) / kRangeBin_m)); + stats.range_histogram[bin]++; + } + + bool writeTrajectory( + const fs::path& file, const Session& session, const Eigen::Vector3d& offset, ExportStats& stats, std::string& error) + { + std::ofstream out(file); + if (!out) + { + error = "Could not create the trajectory file '" + file.string() + "'."; + return false; + } + out << "timestamp_unix_s,x,y,z,qw,qx,qy,qz,om_deg,fi_deg,ka_deg,scan_index,host_clock_unix_s,imu_om_deg,imu_fi_deg,imu_ka_" + "deg\n"; + const auto& clouds = session.point_clouds_container.point_clouds; + char line[512]; + for (size_t i = 0; i < clouds.size(); i++) + { + for (const auto& node : clouds[i].local_trajectory) + { + // As save_trajectories() in step 2: scan pose x node pose, plus the session offset. + Eigen::Affine3d pose = clouds[i].m_pose * node.m_pose; + pose.translation() += offset; + const Eigen::Quaterniond q(pose.rotation()); + const TaitBryanPose tb = pose_tait_bryan_from_affine_matrix(pose); + // node.timestamps.second is the IMU file's timestampUnix column: the scanner computer's clock in + // milliseconds (step 1 passes it through unchanged). + std::snprintf( + line, + sizeof(line), + "%.6f,%.4f,%.4f,%.4f,%.9f,%.9f,%.9f,%.9f,%.6f,%.6f,%.6f,%zu,%.3f,%.6f,%.6f,%.6f\n", + node.timestamps.first / 1e9, + pose.translation().x(), + pose.translation().y(), + pose.translation().z(), + q.w(), + q.x(), + q.y(), + q.z(), + rad2deg(tb.om), + rad2deg(tb.fi), + rad2deg(tb.ka), + i, + node.timestamps.second / 1e3, + rad2deg(node.imu_om_fi_ka.x()), + rad2deg(node.imu_om_fi_ka.y()), + rad2deg(node.imu_om_fi_ka.z())); + out << line; + stats.trajectory_nodes++; + } + } + out.close(); + if (!out) + { + error = "Could not write the trajectory file '" + file.string() + "' (is the disk full?)."; + return false; + } + return true; + } + + std::string formatCount(uint64_t n) + { + std::string s = std::to_string(n); + for (int i = static_cast(s.size()) - 3; i > 0; i -= 3) + s.insert(static_cast(i), ","); + return s; + } + + bool writeReadme(const ExportRequest& request, const ExportStats& stats, std::string& error) + { + std::ofstream out(request.readme_file); + if (!out) + { + error = "Could not create '" + request.readme_file.string() + "'."; + return false; + } + const std::string laz = request.laz_file.filename().string(); + const std::string csv = request.trajectory_file.filename().string(); + char inner[32], near_min[32], output_filter[32], outer[32]; + std::snprintf(inner, sizeof(inner), "%.2f m", stats.filter_threshold_xy_inner); + std::snprintf(outer, sizeof(outer), "%.1f m", stats.filter_threshold_xy_outer); + std::snprintf(near_min, sizeof(near_min), "%.2f m", request.near_min_range_m); + std::snprintf(output_filter, sizeof(output_filter), "%.2f m", stats.threshould_output_filter); + + out << "Easy Forest HDMapping results\n" + << "==========================\n\n" + << "Made by " << versionString() << " from the recording\n" + << " " << request.recording.laz_files.front().parent_path().string() << "\n" + << "with lidar odometry (precision forestry settings). No loop closure and no georeferencing: coordinates\n" + << "are in metres in a local frame that starts at the scanner's first position (z up, from the IMU).\n\n" + << "Files\n-----\n" + << " " << laz << " point cloud (LAZ, LAS 1.4, point format 6), " + << formatCount(stats.odometry_points + stats.near_points + stats.gap_points + stats.check_band_points) << " points\n" + << " " << csv << " trajectory of the scanner (CSV), " << formatCount(stats.trajectory_nodes) << " poses\n" + << " easy_gui.log processing log\n" + << " work/lio_session/ lidar odometry session; open session.mjs in multi_view_tls_registration_step_2\n" + << " to refine or georeference the result\n\n" + << "Time\n----\n" + << "All times are Unix epoch seconds (UTC, seconds since 1970-01-01) as counted by the scanner's lidar\n" + << "clock. They are NOT GPS week time and NOT adjusted standard GPS time, even though the LAS field is called\n" + << "gps_time (the LAS header's global encoding bit 0 is 0 accordingly).\n" + << (request.place_by_raw_time + ? "Every point carries its raw measurement time (experimental \"place by raw time\" export option).\n\n" + : "Lidar odometry's points (point_origin = 0) carry lidar odometry's time, not the raw measurement time:\n" + "within each 0.1 s piece of the recording, odometry spreads its points' times evenly between the first and\n" + "last pose of that piece. That is close to the measurement time (typically 1-4 ms off, at most about\n" + "20 ms), but it cannot be used to match points to the raw files. Near points and gap points\n" + "(point_origin = 1, 2) carry their raw measurement time.\n\n") + << "Point cloud fields (" << laz << ")\n" + << std::string(21 + laz.size(), '-') << "\n" + << " X, Y, Z position [m], 0.1 mm resolution\n" + << " intensity lidar reflectivity as recorded (Livox: 0-255; usually 0-160)\n" + << " gps_time time [Unix epoch seconds], see \"Time\"\n" + << " point_source_id index of the lidar odometry scan (scan_lio_.laz in work/lio_session, and\n" + << " scan_index in " << csv << "); about 5 s of recording each\n" + << " user_data lidar id (" << request.lidar_id << " for every point of this single-lidar recording)\n" + << " return_number, number_of_returns always 1; classification always 0 (not classified)\n" + << " range_m extra bytes, float32: 3D distance from the lidar at the moment of measurement [m];\n" + << " can exceed " << outer << " for steep points, as that limit is horizontal\n" + << " point_origin extra bytes, uint8: where the point comes from (see below)\n" + << (request.place_by_raw_time ? " 0 = inside lidar odometry's distance limits, placed by raw time\n" + : " 0 = lidar odometry's own point\n") + << " 1 = near point, 2 = gap point (both added by Easy Forest HDMapping)\n" + << (stats.check_band_points ? " 3 = TEST MODE alignment-check point (duplicate of an odometry point)\n" + : "") + << "\n" + << "Where the points come from\n--------------------------\n" + << "Lidar odometry only uses and exports points whose horizontal distance from the lidar (in the lidar's own\n" + << "frame) is between " << inner << " and " << outer << " and whose 3D distance is above " << output_filter + << ", and it works in\n" + << "0.1 s pieces: it skips the raw points measured after the last pose of a piece and before the first pose\n" + << "of the next one (about 5 % of the recording time). Easy Forest HDMapping adds both kinds of missing points back from\n" + << "the raw recording, so the cloud has no hole around the walking path and nothing is missing between the\n" + << "pieces.\n\n" + << "point_origin = 0 (" << formatCount(stats.odometry_points) << " points): " + << (request.place_by_raw_time ? "points inside lidar odometry's limits, taken from the raw recording.\n" + : "lidar odometry's own points, as it placed them.\n") + << "point_origin = 1, near points (" << formatCount(stats.near_points) << "): raw points that\n" + << " - are at least " << near_min << " from the lidar in 3D (closer returns are mostly the operator's body and the\n" + << " scanner handle; the limit is the advanced setting \"Closest points kept\"), and\n" + << " - lidar odometry left out because of distance: horizontal distance at most " << inner << ", or 3D\n" + << " distance at most " << output_filter << ", and horizontal distance below " << outer << ".\n" + << "point_origin = 2, gap points (" << formatCount(stats.gap_points) << "): raw points inside lidar odometry's distance\n" + << " limits that it skipped between two 0.1 s pieces.\n" + << "Near and gap points are placed with the trajectory pose lidar odometry uses for points measured at the\n" + << "same time (the last pose before the point), so they line up with lidar odometry's points" + << (request.place_by_raw_time + ? ".\n" + : " to within a few\nmillimetres (rarely more than 2 cm, because of lidar odometry's evenly spread times).\n") + << (request.place_by_raw_time + ? "With the experimental \"place by raw time\" option, point_origin = 0 points are placed the same way.\n" + : "") + << "Points at exactly (0, 0, 0) or with time 0 are \"no return\" records and are never exported, and points\n" + << "measured where the trajectory has no pose within 10 ms (start and end of the recording) are left out.\n\n" + << "Trajectory fields (" << csv << ", comma-separated, one header row)\n" + << std::string(53 + csv.size(), '-') << "\n" + << " timestamp_unix_s time of the pose [Unix epoch seconds], same clock as the point cloud's gps_time\n" + << " x, y, z position of the scanner (IMU) [m], same frame as the point cloud\n" + << " qw, qx, qy, qz orientation as a unit quaternion (rotates scanner coordinates into the frame)\n" + << " om_deg, fi_deg, ka_deg the same orientation as HDMapping's Tait-Bryan angles [deg]\n" + << " scan_index lidar odometry scan the pose belongs to (point_source_id in the point cloud)\n" + << " host_clock_unix_s the scanner computer's own clock at that IMU sample [Unix epoch seconds,\n" + << " millisecond resolution]; it is not synchronised with timestamp_unix_s\n" + << " imu_om_deg, imu_fi_deg, imu_ka_deg orientation from the IMU alone, before lidar odometry [deg]\n"; + out.close(); + if (!out) + { + error = "Could not write '" + request.readme_file.string() + "'."; + return false; + } + return true; + } + } // namespace + + fs::path findSessionParameterFile(const fs::path& session_folder) + { + std::error_code ec; + fs::path found; + for (const auto& entry : fs::directory_iterator(session_folder, ec)) + { + const std::string name = entry.path().filename().string(); + if (name.starts_with("HDMapping_params_") && entry.path().extension() == ".toml" && (found.empty() || entry.path() > found)) + found = entry.path(); + } + return found; + } + + bool exportSession( + const ExportRequest& request, + const std::atomic& cancel, + const ExportProgress& progress, + ExportStats& stats, + std::string& error) + { + stats = ExportStats{}; + const auto cancelled = [&]() + { + if (cancel) + error = "Export was cancelled."; + return cancel.load(); + }; + + // --- the parameters step 1 ran with: its distance filters define which raw points are "near points" + const fs::path parameter_file = findSessionParameterFile(request.session_file.parent_path()); + auto params = std::make_unique(); + TomlIO toml_io; + if (parameter_file.empty() || !toml_io.LoadParametersFromTomlFile(parameter_file.string(), *params)) + { + error = "The lidar odometry result is incomplete: its settings file (HDMapping_params_*.toml) is missing in '" + + request.session_file.parent_path().string() + "'."; + return false; + } + stats.filter_threshold_xy_inner = params->filter_threshold_xy_inner; + stats.filter_threshold_xy_outer = params->filter_threshold_xy_outer; + stats.threshould_output_filter = params->threshould_output_filter; + const int poses_per_piece = params->threshold_nr_poses; + stats.threshold_nr_poses = poses_per_piece; + if (poses_per_piece <= 0) + { + error = "The lidar odometry settings file has no valid threshold_nr_poses."; + return false; + } + + // --- poses and trajectories exactly as step 2 loads them; the points are streamed below instead + progress(0.0, "Reading the lidar odometry result"); + Session session; + session.load_cache_mode = true; + if (!session.load(request.session_file.string(), false, 0.0, 0.0, 0.0, false) || + session.point_clouds_container.point_clouds.empty()) + { + error = "Could not read the lidar odometry result '" + request.session_file.string() + "'."; + return false; + } + const auto& clouds = session.point_clouds_container.point_clouds; + const Eigen::Vector3d offset = session.point_clouds_container.offset; + stats.scans = clouds.size(); + if (clouds.size() > 65535) + { + error = "The recording is too long for one point cloud file (more than 65535 lidar odometry scans)."; + return false; + } + + // --- calibration, as load_data builds it: lidar frame -> the frame step 1 works in (identity without a file) + Eigen::Affine3d calibration = Eigen::Affine3d::Identity(); + { + const auto id_to_sn = MLvxCalib::GetIdToSnMapping(request.recording.sn_file.string()); + const auto combined = MLvxCalib::CombineIntoCalibration( + id_to_sn, + request.recording.calibration_file.empty() + ? std::unordered_map{} + : MLvxCalib::GetCalibrationFromFile(request.recording.calibration_file.string())); + if (const auto it = combined.find(request.lidar_id); it != combined.end()) + calibration = it->second; + } + const Eigen::Affine3d lidar_from_sensor = calibration.inverse(); + + // --- all trajectory nodes in time order, for placing the near points + std::vector nodes; + for (size_t i = 0; i < clouds.size(); i++) + { + if (clouds[i].local_trajectory.empty()) + { + error = "The lidar odometry result is incomplete: scan " + std::to_string(i) + " has no trajectory."; + return false; + } + // Every step 1 piece has exactly threshold_nr_poses nodes, and save_result concatenates whole pieces into a + // scan, so the piece boundaries are at multiples of it within each scan. + const auto& trajectory = clouds[i].local_trajectory; + for (size_t j = 0; j < trajectory.size(); j++) + nodes.push_back( + { trajectory[j].timestamps.first / 1e9, + i, + clouds[i].m_pose * trajectory[j].m_pose, + static_cast(j % poses_per_piece) == poses_per_piece - 1 }); + } + std::stable_sort( + nodes.begin(), + nodes.end(), + [](const Node& a, const Node& b) + { + return a.time_s < b.time_s; + }); + std::vector node_times(nodes.size()); + for (size_t i = 0; i < nodes.size(); i++) + node_times[i] = nodes[i].time_s; + + // --- output files + std::error_code ec; + fs::create_directories(request.laz_file.parent_path(), ec); + Las14Writer writer; + if (!writer.open( + request.laz_file.string(), + kAttributes, + offset, + kCoordinateScale, + // Both header fields hold 32 characters, too few for versionString(): one name each. + std::string("HDMapping ") + HDMAPPING_CONFIGURE_PROJECT_VERSION, + std::string("Easy Forest HDMapping ") + kEasyForestVersion, + error)) + { + error = "Could not create the point cloud file '" + request.laz_file.string() + "' (" + error + ")."; + return false; + } + const uint8_t user_data = static_cast(request.lidar_id); + const size_t scans_to_copy = request.place_by_raw_time ? 0 : clouds.size(); + const double work_units = static_cast(scans_to_copy + request.recording.pairs); + auto writePoint = + [&](const Eigen::Vector3d& position, double time_s, uint16_t intensity, size_t scan, double range, PointOrigin origin) + { + const ExtraBytes extra{ static_cast(range), origin }; + Las14Point p; + p.position = position; + p.gps_time = time_s; + p.intensity = intensity; + p.point_source_id = static_cast(scan); + p.user_data = user_data; + p.extra_bytes = reinterpret_cast(&extra); + addRange(stats, range); + (origin == kStep1 ? stats.odometry_points + : origin == kNearPoint ? stats.near_points + : origin == kGapPoint ? stats.gap_points + : stats.check_band_points)++; + return writer.write(p, error); + }; + + // --- 1. step 1's points, one scan at a time (not with place_by_raw_time: pass 2 places them from the raw files) + for (size_t i = 0; i < scans_to_copy; i++) + { + if (cancelled()) + return false; + progress( + static_cast(i) / work_units, + "Writing the point cloud (scan " + std::to_string(i + 1) + " of " + std::to_string(clouds.size()) + ")"); + + const auto& pc = clouds[i]; + std::vector times; // node times in step 1's unit (seconds x 1e9), as the points' gps_time + std::vector sensor_from_local; + for (const auto& n : pc.local_trajectory) + { + times.push_back(n.timestamps.first); + sensor_from_local.push_back(n.m_pose.inverse()); + } + + LazReader reader; + if (!reader.open(pc.file_name, error)) + { + error = "The lidar odometry result is incomplete: " + error + "."; + return false; + } + for (uint64_t k = 0; k < reader.count(); k++) + { + if (!reader.next(error)) + { + error = "Could not read '" + pc.file_name + "': " + error + "."; + return false; + } + const laszip_point& rp = reader.point(); + if (rp.gps_time == 0.0) // step 2's export skips these too + continue; + // Step 2 subtracts the session offset from the loaded points and adds it back after the pose. + const Eigen::Vector3d local = reader.position() - offset; + + // Range: the point in the sensor frame, through the node step 1 used for it (the last node before the + // point's time), then into the lidar frame. + const auto it = std::lower_bound(times.begin(), times.end(), rp.gps_time); + const size_t node = it == times.begin() ? 0 : static_cast(it - times.begin()) - 1; + const double range = (lidar_from_sensor * (sensor_from_local[node] * local)).norm(); + + if (!writePoint(pc.m_pose * local + offset, rp.gps_time / 1e9, rp.intensity, i, range, kStep1)) + { + error = "Could not write the point cloud file (is the disk full?): " + error + "."; + return false; + } + } + } + + // --- 2. near and gap points (with place_by_raw_time: all points) from the raw recording, one file at a time + const double inner = params->filter_threshold_xy_inner; + const double outer = params->filter_threshold_xy_outer; + const double output_filter = params->threshould_output_filter; + for (size_t f = 0; f < request.recording.pairs; f++) + { + if (cancelled()) + return false; + progress( + static_cast(scans_to_copy + f) / work_units, + std::string(request.place_by_raw_time ? "Writing the point cloud" : "Adding the near and gap points") + " (file " + + std::to_string(f + 1) + " of " + std::to_string(request.recording.pairs) + ")"); + + const fs::path& raw_file = request.recording.laz_files[f]; + LazReader reader; + if (!reader.open(raw_file, error)) + { + error = "Could not read the recording file '" + raw_file.filename().string() + "'."; + return false; + } + for (uint64_t k = 0; k < reader.count(); k++) + { + if (!reader.next(error)) + { + error = "Could not read the recording file '" + raw_file.filename().string() + "': " + error + "."; + return false; + } + stats.raw_points_read++; + const laszip_point& rp = reader.point(); + if ((rp.X == 0 && rp.Y == 0 && rp.Z == 0) || rp.gps_time == 0.0) + { + stats.raw_no_return++; + continue; + } + const Eigen::Vector3d lidar = reader.position(); + const double range_xy = std::hypot(lidar.x(), lidar.y()); + const double range = lidar.norm(); + const Eigen::Vector3d sensor = calibration * lidar; + + // Step 1's distance filters: load_point_cloud keeps inner < range_xy < outer, points_to_vector keeps + // |sensor| > threshould_output_filter. Near points are exactly the complement, at close range. + const bool step1_keeps = range_xy > inner && range_xy < outer && sensor.norm() > output_filter; + if (!step1_keeps && (range < request.near_min_range_m || range_xy >= outer)) + continue; + + const double t = rp.gps_time; + const auto it = std::lower_bound(node_times.begin(), node_times.end(), t); + if (it == node_times.begin() || t - *(it - 1) > kNodeTolerance_s) + { + if (!step1_keeps || request.place_by_raw_time) + stats.raw_outside_trajectory++; + continue; + } + // Placed with the last node before the point (step 1's rule). Whether step 1 has the point depends on + // the last node at or before it: a piece's last node starts a gap step 1 skips. + const Node& node = nodes[static_cast(it - node_times.begin()) - 1]; + const size_t at_or_before = + static_cast(std::upper_bound(node_times.begin(), node_times.end(), t) - node_times.begin()) - 1; + const bool in_gap = nodes[at_or_before].piece_end; + + PointOrigin origin; + if (!step1_keeps) + origin = kNearPoint; + else if (in_gap) + origin = kGapPoint; + else if (request.place_by_raw_time) + origin = kStep1; + else if (range_xy < request.check_band_outer_m) + origin = kCheckBand; + else + continue; // step 1 has this point: exported from scan_lio_N.laz in pass 1 + + if (!writePoint(node.global * sensor + offset, t, rp.intensity, node.scan, range, origin)) + { + error = "Could not write the point cloud file (is the disk full?): " + error + "."; + return false; + } + } + } + + if (!writer.close(error)) + { + error = "Could not finish the point cloud file (is the disk full?): " + error + "."; + return false; + } + stats.laz_bytes = fs::file_size(request.laz_file, ec); + + progress(1.0, "Writing the trajectory"); + if (!writeTrajectory(request.trajectory_file, session, offset, stats, error)) + return false; + stats.trajectory_bytes = fs::file_size(request.trajectory_file, ec); + return writeReadme(request, stats, error); + } +} // namespace easy_gui diff --git a/apps/easy_gui/src/export.h b/apps/easy_gui/src/export.h new file mode 100644 index 00000000..ea7942fe --- /dev/null +++ b/apps/easy_gui/src/export.h @@ -0,0 +1,85 @@ +#pragma once + +#include "preflight.h" + +#include +#include +#include +#include +#include +#include + +namespace easy_gui +{ + //! Values of point_origin, the extra-bytes field of every exported point. + enum PointOrigin : uint8_t + { + kStep1 = 0, // lidar odometry's (step 1's) own point, placed by step 1 + kNearPoint = 1, // raw point step 1's distance filters removed, added by Easy GUI at its real time + kGapPoint = 2, // raw point inside step 1's limits that step 1 skipped between its 0.1 s pieces, added at its real time + kCheckBand = 3, // test mode only: raw point step 1 also exported, added again to check the alignment + }; + + struct ExportRequest + { + std::filesystem::path session_file; // step 1 session.mjs + RecordingFiles recording; // the raw files step 1 read (near points come from them) + int lidar_id = 0; // user_data of the raw points (single-lidar recordings only) + double near_min_range_m = 0.4; // near points closer than this (3D) are left out + //! Test mode: > 0 also adds the raw points with filter_threshold_xy_inner < horizontal range < this, which step 1 + //! exported too, as kCheckBand, so their placement can be compared with step 1's own points. + double check_band_outer_m = 0.0; + //! Experimental: place every point from the raw files with the trajectory pose at its real gps_time, instead of + //! taking step 1's points (scan_lio_N.laz, placed at step 1's ramp time). Same distance filters, near-point and + //! gap-point rules; raw gps_time is written. point_origin = 0 then means "inside step 1's distance limits, and + //! not in a gap". check_band_outer_m is ignored (those points are all included). Measured slightly less sharp + //! than the default (see docs/DESIGN.md), hence off. + bool place_by_raw_time = false; + std::filesystem::path laz_file; + std::filesystem::path trajectory_file; + std::filesystem::path readme_file; + }; + + struct ExportStats + { + size_t scans = 0; + size_t trajectory_nodes = 0; + uint64_t odometry_points = 0; // point_origin = 0 + uint64_t near_points = 0; // point_origin = 1 + uint64_t gap_points = 0; // point_origin = 2 + uint64_t check_band_points = 0; // point_origin = 3 + uint64_t raw_points_read = 0; + uint64_t raw_no_return = 0; // (0,0,0) or gps_time 0: never exported + uint64_t raw_outside_trajectory = 0; // exported candidates with no trajectory node within 10 ms + double range_min_m = 0.0; + double range_max_m = 0.0; + //! range_m histogram, 0.1 m bins from 0 m; the last bin collects everything beyond. + std::array range_histogram{}; + uint64_t laz_bytes = 0; + uint64_t trajectory_bytes = 0; + double filter_threshold_xy_inner = 0.0; + double filter_threshold_xy_outer = 0.0; + double threshould_output_filter = 0.0; + int threshold_nr_poses = 0; // trajectory poses per step 1 piece (worker chunk) + }; + + //! Called with the fraction done (0..1) and a short description of the current step. + using ExportProgress = std::function; + + //! Writes the Easy GUI results of one step 1 session: + //! - laz_file: LAS 1.4 point format 6 LAZ with all step 1 points plus the near points (see README.txt), in + //! step 1's coordinate frame, poses as step 2 applies them (session_poses.mrp x the scan's points); + //! - trajectory_file: CSV, one row per trajectory node; + //! - readme_file: plain-text description of both files. + //! Streams one scan (step 1 chunk) or one raw file at a time, so memory does not grow with the recording. + //! Returns false with a plain-language `error` on failure, or when `cancel` became true. + bool exportSession( + const ExportRequest& request, + const std::atomic& cancel, + const ExportProgress& progress, + ExportStats& stats, + std::string& error); + + //! The step 1 parameter file in a session folder (HDMapping_params_*.toml, written by save_result), or empty. + std::filesystem::path findSessionParameterFile(const std::filesystem::path& session_folder); +} // namespace easy_gui diff --git a/apps/easy_gui/src/forestry_preset.cpp b/apps/easy_gui/src/forestry_preset.cpp new file mode 100644 index 00000000..e2aaa57a --- /dev/null +++ b/apps/easy_gui/src/forestry_preset.cpp @@ -0,0 +1,58 @@ +#include "forestry_preset.h" + +#include + +#include + +namespace easy_gui +{ + void applyPrecisionForestryPreset(LidarOdometryParams& params) + { + params.decimation = 0.01; + params.in_out_params_indoor.resolution_X = 0.1; + params.in_out_params_indoor.resolution_Y = 0.1; + params.in_out_params_indoor.resolution_Z = 0.1; + + params.in_out_params_outdoor.resolution_X = 0.3; + params.in_out_params_outdoor.resolution_Y = 0.3; + params.in_out_params_outdoor.resolution_Z = 0.3; + + params.filter_threshold_xy_inner = 1.0; + params.filter_threshold_xy_outer = 70.0; + params.threshould_output_filter = 1.5; + + params.use_robust_and_accurate_lidar_odometry = false; + params.distance_bucket = 0.2; + params.polar_angle_deg = 10.0; + params.azimutal_angle_deg = 10.0; + params.robust_and_accurate_lidar_odometry_iterations = 20; + + params.max_distance_lidar = 70.0; + params.nr_iter = 500; + params.sliding_window_trajectory_length_threshold = 10000; + params.real_time_threshold_seconds = 10; + } + + void applyEasyGuiOutputSettings(LidarOdometryParams& params) + { + params.threshould_output_filter = params.filter_threshold_xy_inner; + } + + bool writeStep1ParameterFile(const std::filesystem::path& file, std::string& error) + { + // Heap + value-initialisation, as lidar_odometry_gui.cpp's --dump-default-params does: the NDT grid members + // carry no initialisers (and would be written to the TOML), and the struct is too large for comfort on the + // stack of the pipeline thread. + auto params = std::make_unique(); + applyPrecisionForestryPreset(*params); + applyEasyGuiOutputSettings(*params); + + TomlIO toml_io; + if (!toml_io.SaveParametersToTomlFile(file.string(), *params)) + { + error = "Could not write the processing settings file '" + file.string() + "'."; + return false; + } + return true; + } +} // namespace easy_gui diff --git a/apps/easy_gui/src/forestry_preset.h b/apps/easy_gui/src/forestry_preset.h new file mode 100644 index 00000000..7cb110ff --- /dev/null +++ b/apps/easy_gui/src/forestry_preset.h @@ -0,0 +1,26 @@ +#pragma once + +#include +#include + +struct LidarOdometryParams; + +namespace easy_gui +{ + //! "Presets -> 2 Velocity < 8km/h, larger spaces, precise forestry (generic)" of lidar_odometry_step_1. + //! + //! NOTE: the preset only exists as inline GUI code (lidar_odometry_gui.cpp, and a copy in + //! drag_folder_with_mandeye_data_and_drop_here-precision_forestry.cpp), so the values are repeated here. They + //! overwrite every field set_lidar_odometry_default_params() touches, so the result equals "start the step 1 GUI, + //! pick preset 2" regardless of which app's default function ran first. Keep in sync with lidar_odometry_gui.cpp. + void applyPrecisionForestryPreset(LidarOdometryParams& params); + + //! Easy GUI's only deviation from the preset: threshould_output_filter (output-only) is lowered to + //! filter_threshold_xy_inner, so step 1 exports every point the load filter kept, and the near-point export picks + //! up exactly the points the load filter removed. The trajectory is unaffected (see docs/DESIGN.md). + void applyEasyGuiOutputSettings(LidarOdometryParams& params); + + //! Writes the parameter file the step 1 child process runs with (value-initialised params + preset + Easy GUI + //! output settings), through step 1's own TomlIO so every serialisable field is present. + bool writeStep1ParameterFile(const std::filesystem::path& file, std::string& error); +} // namespace easy_gui diff --git a/apps/easy_gui/src/las14_writer.cpp b/apps/easy_gui/src/las14_writer.cpp new file mode 100644 index 00000000..90f5c0cf --- /dev/null +++ b/apps/easy_gui/src/las14_writer.cpp @@ -0,0 +1,209 @@ +#include "las14_writer.h" + +#include + +#include +#include +#include + +namespace easy_gui +{ + namespace + { + //! LAS 1.4 R15 "Extra Bytes" descriptor. Written by hand rather than with laszip_add_attribute(), which always + //! sets the "scale" option bit: readers then scale the values, so laspy would return the uint8 point_origin field as + //! float64. +#pragma pack(push, 1) + struct ExtraBytesDescriptor + { + uint8_t reserved[2]; + uint8_t data_type; + uint8_t options; // 0: no no_data/min/max/scale/offset + char name[32]; + uint8_t unused[4]; + uint8_t no_data[24]; + uint8_t min[24]; + uint8_t max[24]; + double scale[3]; + double offset[3]; + char description[32]; + }; +#pragma pack(pop) + static_assert(sizeof(ExtraBytesDescriptor) == 192); + + size_t attributeSize(ExtraBytesAttribute::Type type) + { + return type == ExtraBytesAttribute::Type::U8 ? 1 : 4; + } + + std::string laszipError(void* writer, const std::string& what) + { + laszip_CHAR* message = nullptr; + if (writer && !laszip_get_error(writer, &message) && message && *message) + return what + ": " + message; + return what; + } + + void copyField(char* destination, size_t size, const std::string& text) + { + std::memset(destination, 0, size); + std::memcpy(destination, text.data(), std::min(size, text.size())); + } + + constexpr uint16_t kPointFormat6RecordLength = 30; + } // namespace + + Las14Writer::~Las14Writer() + { + std::string ignored; + close(ignored); + } + + bool Las14Writer::open( + const std::string& file_name, + const std::vector& attributes, + const Eigen::Vector3d& offset, + double scale, + const std::string& system_identifier, + const std::string& generating_software, + std::string& error) + { + if (laszip_create(&writer_)) + { + writer_ = nullptr; + error = "could not create the LAZ writer"; + return false; + } + + laszip_header* header = nullptr; + if (laszip_get_header_pointer(writer_, &header)) + { + error = laszipError(writer_, "could not access the LAZ header"); + return false; + } + + extra_bytes_size_ = 0; + for (const auto& a : attributes) + extra_bytes_size_ += attributeSize(a.type); + + header->file_source_ID = 0; + header->global_encoding = (1 << 4); // bit 0 = 0: not adjusted GPS time; bit 4: WKT (required for PF6) + header->version_major = 1; + header->version_minor = 4; + copyField(header->system_identifier, sizeof(header->system_identifier), system_identifier); + copyField(header->generating_software, sizeof(header->generating_software), generating_software); + { + const std::time_t now = std::time(nullptr); + std::tm tm{}; +#ifdef _WIN32 + gmtime_s(&tm, &now); +#else + gmtime_r(&now, &tm); +#endif + header->file_creation_day = static_cast(tm.tm_yday + 1); + header->file_creation_year = static_cast(tm.tm_year + 1900); + } + header->header_size = 375; + header->offset_to_point_data = 375; // laszip_add_vlr() below adds the VLR sizes + header->point_data_format = 6; + header->point_data_record_length = static_cast(kPointFormat6RecordLength + extra_bytes_size_); + header->x_scale_factor = header->y_scale_factor = header->z_scale_factor = scale; + header->x_offset = offset.x(); + header->y_offset = offset.y(); + header->z_offset = offset.z(); + + if (!attributes.empty()) + { + std::vector descriptors(attributes.size()); + for (size_t i = 0; i < attributes.size(); i++) + { + ExtraBytesDescriptor& d = descriptors[i]; + std::memset(&d, 0, sizeof(d)); + d.data_type = static_cast(attributes[i].type); + copyField(d.name, sizeof(d.name), attributes[i].name); + copyField(d.description, sizeof(d.description), attributes[i].description); + } + if (laszip_add_vlr( + writer_, + "LASF_Spec", + 4, + static_cast(descriptors.size() * sizeof(ExtraBytesDescriptor)), + "extra bytes", + reinterpret_cast(descriptors.data()))) + { + error = laszipError(writer_, "could not add the extra bytes record"); + return false; + } + } + + laszip_preserve_generating_software(writer_, 1); + const laszip_BOOL compress = file_name.size() >= 4 && file_name.compare(file_name.size() - 4, 4, ".laz") == 0; + if (laszip_open_writer(writer_, file_name.c_str(), compress)) + { + error = laszipError(writer_, "could not open '" + file_name + "' for writing"); + return false; + } + + laszip_point* point = nullptr; + if (laszip_get_point_pointer(writer_, &point)) + { + error = laszipError(writer_, "could not access the LAZ point"); + return false; + } + if (static_cast(point->num_extra_bytes) != extra_bytes_size_) + { + error = "LAZ writer has " + std::to_string(point->num_extra_bytes) + " extra bytes per point, expected " + + std::to_string(extra_bytes_size_); + return false; + } + point->extended_point_type = 1; + point->return_number = 1; + point->number_of_returns = 1; + point->extended_return_number = 1; + point->extended_number_of_returns = 1; + point_ = point; + points_written_ = 0; + return true; + } + + bool Las14Writer::write(const Las14Point& p, std::string& error) + { + auto* point = static_cast(point_); + point->gps_time = p.gps_time; + point->intensity = p.intensity; + point->point_source_ID = p.point_source_id; + point->user_data = p.user_data; + if (extra_bytes_size_ > 0) + { + if (p.extra_bytes) + std::memcpy(point->extra_bytes, p.extra_bytes, extra_bytes_size_); + else + std::memset(point->extra_bytes, 0, extra_bytes_size_); + } + + laszip_F64 coordinates[3] = { p.position.x(), p.position.y(), p.position.z() }; + if (laszip_set_coordinates(writer_, coordinates) || laszip_write_point(writer_) || laszip_update_inventory(writer_)) + { + error = laszipError(writer_, "writing point " + std::to_string(points_written_)); + return false; + } + points_written_++; + return true; + } + + bool Las14Writer::close(std::string& error) + { + if (!writer_) + return true; + bool ok = true; + if (point_ && laszip_close_writer(writer_)) + { + error = laszipError(writer_, "could not finish the LAZ file"); + ok = false; + } + laszip_destroy(writer_); + writer_ = nullptr; + point_ = nullptr; + return ok; + } +} // namespace easy_gui diff --git a/apps/easy_gui/src/las14_writer.h b/apps/easy_gui/src/las14_writer.h new file mode 100644 index 00000000..15646486 --- /dev/null +++ b/apps/easy_gui/src/las14_writer.h @@ -0,0 +1,74 @@ +#pragma once + +#include + +#include +#include +#include + +namespace easy_gui +{ + //! One named extra-bytes attribute (LAS 1.4 "Extra Bytes" VLR, record id 4). + struct ExtraBytesAttribute + { + enum class Type : uint8_t + { + U8 = 1, // LAS 1.4 data_type codes + F32 = 9 + }; + Type type; + std::string name; // max 32 characters + std::string description; // max 32 characters + }; + + struct Las14Point + { + Eigen::Vector3d position; // in the file's coordinate frame, before header offset/scale + double gps_time = 0.0; + uint16_t intensity = 0; + uint16_t point_source_id = 0; + uint8_t user_data = 0; + const uint8_t* extra_bytes = nullptr; // packed attribute values in declaration order, or null for all zeros + }; + + //! Streaming LAZ writer for LAS 1.4, point format 6, with extra-bytes attributes. Points are written one by one; + //! header counts and bounds are filled in at close() (laszip inventory), so nothing is buffered here. + //! + //! Global encoding: bit 0 (GPS time type) = 0, because gps_time holds Unix epoch seconds, not adjusted standard + //! GPS time; bit 4 (WKT) = 1, which LAS 1.4 requires for point formats 6-10 (no CRS record is written). + class Las14Writer + { + public: + Las14Writer() = default; + ~Las14Writer(); + Las14Writer(const Las14Writer&) = delete; + Las14Writer& operator=(const Las14Writer&) = delete; + + //! `scale` is the coordinate resolution in metres; `offset` the header offset. + bool open( + const std::string& file_name, + const std::vector& attributes, + const Eigen::Vector3d& offset, + double scale, + const std::string& system_identifier, + const std::string& generating_software, + std::string& error); + bool write(const Las14Point& point, std::string& error); + bool close(std::string& error); + + uint64_t pointsWritten() const + { + return points_written_; + } + size_t extraBytesSize() const + { + return extra_bytes_size_; + } + + private: + void* writer_ = nullptr; // laszip_POINTER + void* point_ = nullptr; // laszip_point* + size_t extra_bytes_size_ = 0; + uint64_t points_written_ = 0; + }; +} // namespace easy_gui diff --git a/apps/easy_gui/src/pipeline.cpp b/apps/easy_gui/src/pipeline.cpp new file mode 100644 index 00000000..8444ee9a --- /dev/null +++ b/apps/easy_gui/src/pipeline.cpp @@ -0,0 +1,408 @@ +#include "pipeline.h" + +#include "forestry_preset.h" +#include "staging.h" +#include "version.h" + +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; + +namespace easy_gui +{ + namespace + { + constexpr size_t kLogTailLines = 400; + + std::string timestamp() + { + const std::time_t now = std::time(nullptr); + std::tm tm{}; +#ifdef _WIN32 + localtime_s(&tm, &now); +#else + localtime_r(&now, &tm); +#endif + std::ostringstream out; + out << std::put_time(&tm, "%Y-%m-%d %H:%M:%S"); + return out.str(); + } + + const char* severityTag(Severity s) + { + return s == Severity::Error ? "ERROR" : s == Severity::Warning ? "WARNING" : "OK"; + } + } // namespace + + OutputLayout::OutputLayout(const fs::path& root_) + : root(root_) + , log_file(root_ / "easy_gui.log") + , work(root_ / "work") + , staging(root_ / "work" / "staging") + , step1_parameters(root_ / "work" / "easy_gui_step1.toml") + , session(root_ / "work" / "lio_session") + , point_cloud(root_ / "point_cloud.laz") + , trajectory(root_ / "trajectory.csv") + , readme(root_ / "README.txt") + , done_marker(root_ / "easy_gui_done.txt") + { + } + + fs::path findStep1Executable() + { + if (const char* overridden = std::getenv("HDMAPPING_STEP1_EXECUTABLE"); overridden && *overridden) + return fs::path(overridden); + + const fs::path dir = currentExecutableDirectory(); +#ifdef _WIN32 + const char* suffix = ".exe"; +#else + const char* suffix = ""; +#endif + for (const std::string name : + { std::string("lidar_odometry_step_1"), std::string(HDMAPPING_BINARY_PREFIX) + "lidar_odometry_step_1" }) + { + const fs::path candidate = dir / (name + suffix); + std::error_code ec; + if (fs::is_regular_file(candidate, ec)) + return candidate; + } + return {}; + } + + Pipeline::~Pipeline() + { + cancel(); + if (thread_.joinable()) + thread_.join(); + } + + bool Pipeline::start(const PipelineRequest& request, std::string& error) + { + if (busy_) + { + error = "Processing is already running."; + return false; + } + if (thread_.joinable()) + thread_.join(); + + request_ = request; + request_.settings.clamp(); + cancel_requested_ = false; + { + std::lock_guard lock(mutex_); + status_ = PipelineStatus{}; + status_.state = PipelineState::Running; + log_tail_.clear(); + started_ = std::chrono::steady_clock::now(); + } + busy_ = true; + thread_ = std::thread(&Pipeline::run, this); + return true; + } + + void Pipeline::cancel() + { + cancel_requested_ = true; + std::lock_guard lock(mutex_); + if (child_) + child_->kill(); + } + + void Pipeline::reset() + { + if (busy_) + return; + if (thread_.joinable()) + thread_.join(); + std::lock_guard lock(mutex_); + status_ = PipelineStatus{}; + log_tail_.clear(); + } + + PipelineStatus Pipeline::status() const + { + std::lock_guard lock(mutex_); + PipelineStatus s = status_; + if (s.state == PipelineState::Running) + s.elapsed_s = std::chrono::duration(std::chrono::steady_clock::now() - started_).count(); + return s; + } + + std::vector Pipeline::logTail() const + { + std::lock_guard lock(mutex_); + return { log_tail_.begin(), log_tail_.end() }; + } + + void Pipeline::log(const std::string& line) + { + if (log_file_.is_open()) + log_file_ << line << '\n'; + std::lock_guard lock(mutex_); + log_tail_.push_back(line); + while (log_tail_.size() > kLogTailLines) + log_tail_.pop_front(); + } + + void Pipeline::setStage(PipelineStage stage) + { + std::lock_guard lock(mutex_); + status_.stage = stage; + } + + void Pipeline::finish(PipelineState state, const std::string& message, const std::string& detail) + { + log("[easy_gui] " + timestamp() + " " + (state == PipelineState::Succeeded ? "finished: " : "stopped: ") + message); + if (!detail.empty()) + log("[easy_gui] detail: " + detail); + log_file_.close(); // finish() is the last thing a run logs + std::lock_guard lock(mutex_); + status_.state = state; + status_.message = message; + status_.detail = detail; + status_.elapsed_s = std::chrono::duration(std::chrono::steady_clock::now() - started_).count(); + } + + void Pipeline::run() + { + const OutputLayout out(request_.output_folder); + std::error_code ec; + + auto cleanupWork = [&](bool success) + { + setStage(PipelineStage::Cleanup); + if (request_.settings.keep_work_files) + return; + // The session stays on success: the export reads it, and an expert can open it in step 2. Everything + // else in work/ (links, step 1's cache/ and preview/) is disposable, and the cache can be gigabytes. + fs::remove_all(success ? out.staging : out.work, ec); + }; + + // --- 1. re-check: the recording or the disk may have changed since the user pressed "Check" + setStage(PipelineStage::Checking); + const PreflightResult check = runPreflight(request_.recording, request_.output_folder, request_.resume_unfinished); + const bool replace_unfinished = request_.resume_unfinished && holdsUnfinishedOutput(out.root); + if (!check.ok()) + { + std::string first_error; + for (const auto& f : check.findings) + if (f.severity == Severity::Error) + { + first_error = f.message; + break; + } + finish(PipelineState::Failed, first_error); + busy_ = false; + return; + } + + if (!fs::create_directories(out.root, ec) && ec) + { + finish(PipelineState::Failed, "Could not create the output folder '" + out.root.string() + "' (" + ec.message() + ")."); + busy_ = false; + return; + } + log_file_.open(out.log_file, std::ios::out | std::ios::app); + + log("[easy_gui] " + versionString() + ", started " + timestamp()); + log("[easy_gui] recording: " + request_.recording.string()); + log("[easy_gui] output: " + out.root.string()); + log("[easy_gui] step 1 program: " + request_.step1_executable.string()); + { + std::ostringstream s; + s << "[easy_gui] settings: near_min_range_m=" << request_.settings.near_min_range_m + << " place_by_raw_time=" << request_.settings.place_by_raw_time << " keep_work_files=" << request_.settings.keep_work_files; + log(s.str()); + } + for (const auto& f : check.findings) + log(std::string("[easy_gui] check ") + severityTag(f.severity) + ": " + f.message); + if (replace_unfinished) + { + // Only Easy GUI's own files are here (holdsUnfinishedOutput); a half-written result must not survive into + // this run, and step 1 needs a fresh staging folder. + std::vector leftovers; + for (const auto& entry : fs::directory_iterator(out.root, ec)) + if (entry.path().filename() != out.log_file.filename()) + leftovers.push_back(entry.path()); + for (const auto& p : leftovers) + fs::remove_all(p, ec); + log("[easy_gui] removed the files of an unfinished earlier attempt"); + } + + // --- 2. staging folder + parameter file + setStage(PipelineStage::Staging); + std::string error; + std::string method; + if (!createStagingFolder(check.files, out.staging, method, error)) + { + cleanupWork(false); + finish(PipelineState::Failed, error); + busy_ = false; + return; + } + log("[easy_gui] staged " + std::to_string(check.files.pairs) + " file pairs as " + method); + if (!writeStep1ParameterFile(out.step1_parameters, error)) + { + cleanupWork(false); + finish(PipelineState::Failed, error); + busy_ = false; + return; + } + + // --- 3. lidar odometry in a child process + setStage(PipelineStage::Odometry); + if (request_.step1_executable.empty() || !fs::is_regular_file(request_.step1_executable, ec)) + { + cleanupWork(false); + finish( + PipelineState::Failed, + "Easy Forest HDMapping could not find the lidar odometry program (lidar_odometry_step_1) next to itself. Please reinstall " + "Easy Forest HDMapping.", + "looked for: " + request_.step1_executable.string()); + busy_ = false; + return; + } + + ChildProcess child; + const std::vector args = { out.staging.string(), out.step1_parameters.string(), out.session.string() }; + log("[easy_gui] running: " + request_.step1_executable.string() + " " + args[0] + " " + args[1] + " " + args[2]); + if (cancel_requested_ || !child.start(request_.step1_executable, args, error)) + { + cleanupWork(false); + if (cancel_requested_) + finish(PipelineState::Cancelled, "Processing was cancelled."); + else + finish(PipelineState::Failed, "The lidar odometry program could not be started.", error); + busy_ = false; + return; + } + { + std::lock_guard lock(mutex_); + child_ = &child; + } + if (cancel_requested_) // cancel() may have run between start() and publishing child_ + child.kill(); + + ProgressParser parser; + std::string line; + while (child.readLine(line)) + { + parser.feed(line); + log(line); + std::lock_guard lock(mutex_); + status_.step1 = parser.progress(); + } + const int exit_code = child.wait(); + { + std::lock_guard lock(mutex_); + child_ = nullptr; + status_.step1 = parser.progress(); + } + + const bool session_written = fs::is_regular_file(out.session / "session.mjs", ec); + if (cancel_requested_ || child.wasKilled()) + { + cleanupWork(false); + finish(PipelineState::Cancelled, "Processing was cancelled. Nothing was kept except the log file."); + } + // NOTE: step 1 exits with 0 after a caught exception, so the session file is the success criterion, not the + // exit code alone. + else if (exit_code != 0 || !session_written || parser.progress().failed) + { + std::string message = parser.progress().failure; + if (message.empty()) + message = exit_code != 0 ? "Lidar odometry stopped unexpectedly (exit code " + std::to_string(exit_code) + ")." + : "Lidar odometry finished without writing a result."; + cleanupWork(false); + finish(PipelineState::Failed, message, parser.progress().last_error_line); + } + else + { + { + std::lock_guard lock(mutex_); + status_.session_folder = out.session; + } + log("[easy_gui] " + timestamp() + " lidar odometry finished, exporting"); + runExport(check, out, cleanupWork); + } + busy_ = false; + } + + void Pipeline::runExport(const PreflightResult& check, const OutputLayout& out, const std::function& cleanupWork) + { + setStage(PipelineStage::Export); + ExportRequest request; + request.session_file = out.session / "session.mjs"; + request.recording = check.files; + request.lidar_id = std::max(0, check.lidar_id); + request.near_min_range_m = request_.settings.near_min_range_m; + request.check_band_outer_m = request_.check_band_outer_m; + request.place_by_raw_time = request_.settings.place_by_raw_time; + request.laz_file = out.point_cloud; + request.trajectory_file = out.trajectory; + request.readme_file = out.readme; + + std::string last_phase; + auto progress = [&](double fraction, const std::string& step) + { + // One log line per phase, not per scan/file: "Writing the point cloud (scan 3 of 113)" -> "Writing the point cloud" + const std::string phase = step.substr(0, step.find(" (")); + if (phase != last_phase) + log("[easy_gui] " + timestamp() + " export: " + phase); + last_phase = phase; + std::lock_guard lock(mutex_); + status_.export_fraction = fraction; + status_.export_step = step; + }; + + ExportStats stats; + std::string error; + const bool ok = exportSession(request, cancel_requested_, progress, stats, error); + std::error_code ec; + if (!ok) + { + for (const auto& f : { out.point_cloud, out.trajectory, out.readme }) + fs::remove(f, ec); + if (cancel_requested_) + { + cleanupWork(false); + finish(PipelineState::Cancelled, "Processing was cancelled. Nothing was kept except the log file."); + return; + } + // NOTE: unlike the other failures, the odometry session stays: it took most of the run, and step 2 can + // open and export it. + cleanupWork(true); + finish( + PipelineState::Failed, + "Lidar odometry finished, but the export failed: " + error + " The lidar odometry result was kept in '" + + out.session.string() + "'."); + return; + } + + { + std::ostringstream s; + s << "[easy_gui] export: " << stats.odometry_points << " odometry points + " << stats.near_points << " near points + " + << stats.gap_points << " gap points"; + if (stats.check_band_points) + s << " + " << stats.check_band_points << " check-band points"; + s << " (" << stats.raw_no_return << " of " << stats.raw_points_read << " raw points were no-return, " + << stats.raw_outside_trajectory << " candidates outside the trajectory), range " << stats.range_min_m << " - " + << stats.range_max_m << " m, " << stats.trajectory_nodes << " trajectory poses; point cloud " << stats.laz_bytes + << " bytes, trajectory " << stats.trajectory_bytes << " bytes"; + log(s.str()); + } + { + std::lock_guard lock(mutex_); + status_.export_fraction = 1.0; + status_.export_stats = stats; + } + cleanupWork(true); + finish(PipelineState::Succeeded, "Done. The point cloud and trajectory are in '" + out.root.string() + "'."); + } +} // namespace easy_gui diff --git a/apps/easy_gui/src/pipeline.h b/apps/easy_gui/src/pipeline.h new file mode 100644 index 00000000..1168b705 --- /dev/null +++ b/apps/easy_gui/src/pipeline.h @@ -0,0 +1,127 @@ +#pragma once + +#include "child_process.h" +#include "export.h" +#include "preflight.h" +#include "progress_parser.h" +#include "settings.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace easy_gui +{ + struct PipelineRequest + { + std::filesystem::path recording; + std::filesystem::path output_folder; + std::filesystem::path step1_executable; + Settings settings; + //! Dev/test only (ExportRequest::check_band_outer_m): > 0 adds alignment-check points to the export. + double check_band_outer_m = 0.0; + //! Batch mode: the output folder may hold an unfinished earlier attempt (holdsUnfinishedOutput); everything in + //! it except easy_gui.log is removed before the run, and the log is continued. + bool resume_unfinished = false; + }; + + enum class PipelineState + { + Idle, + Running, + Succeeded, + Failed, + Cancelled + }; + + enum class PipelineStage + { + Checking, + Staging, + Odometry, + Export, + Cleanup + }; + + struct PipelineStatus + { + PipelineState state = PipelineState::Idle; + PipelineStage stage = PipelineStage::Checking; + Step1Progress step1; + std::string message; // plain-language outcome / failure reason + std::string detail; // technical detail for the "details" view + double elapsed_s = 0.0; + std::filesystem::path session_folder; // step 1 session (session.mjs) once odometry succeeded + double export_fraction = 0.0; // 0..1 during PipelineStage::Export + std::string export_step; // what the export is doing right now + ExportStats export_stats; // filled once the export succeeded + }; + + //! Layout of an Easy GUI output folder. + struct OutputLayout + { + explicit OutputLayout(const std::filesystem::path& root); + std::filesystem::path root; + std::filesystem::path log_file; // easy_gui.log: everything, including the full step 1 output + std::filesystem::path work; // work/: intermediate data + std::filesystem::path staging; // work/staging: links to the recording; step 1 puts cache/ and preview/ here + std::filesystem::path step1_parameters; // work/easy_gui_step1.toml + std::filesystem::path session; // work/lio_session: step 1 output (session.mjs, scan_lio_N.laz, ...) + std::filesystem::path point_cloud; // point_cloud.laz: the exported point cloud + std::filesystem::path trajectory; // trajectory.csv + std::filesystem::path readme; // README.txt: describes the two files above + std::filesystem::path done_marker; // easy_gui_done.txt: written by batch mode once the result is complete + }; + + //! Locates lidar_odometry_step_1 next to the Easy GUI executable (plain build name or the installed, + //! HDMAPPING_BINARY_PREFIX-ed one). The HDMAPPING_STEP1_EXECUTABLE environment variable overrides the search. + std::filesystem::path findStep1Executable(); + + //! Runs the whole Easy GUI processing on a worker thread: re-check, stage, step 1 in a child process, export, clean + //! up. + //! The GUI polls status() every frame; cancel() kills the child process. + class Pipeline + { + public: + ~Pipeline(); + + bool start(const PipelineRequest& request, std::string& error); + void cancel(); + //! Back to Idle after a finished run (no-op while busy). + void reset(); + bool busy() const + { + return busy_.load(); + } + + PipelineStatus status() const; + //! Last lines of the log, oldest first. + std::vector logTail() const; + + private: + void run(); + void runExport(const PreflightResult& check, const OutputLayout& out, const std::function& cleanupWork); + void log(const std::string& line); + void setStage(PipelineStage stage); + void finish(PipelineState state, const std::string& message, const std::string& detail = ""); + + PipelineRequest request_; + std::thread thread_; + std::atomic busy_{ false }; + std::atomic cancel_requested_{ false }; + + mutable std::mutex mutex_; + PipelineStatus status_; + std::deque log_tail_; + std::chrono::steady_clock::time_point started_; + ChildProcess* child_ = nullptr; // valid while the odometry child runs; guarded by mutex_ + std::ofstream log_file_; // only touched by the worker thread + }; +} // namespace easy_gui diff --git a/apps/easy_gui/src/preflight.cpp b/apps/easy_gui/src/preflight.cpp new file mode 100644 index 00000000..070004da --- /dev/null +++ b/apps/easy_gui/src/preflight.cpp @@ -0,0 +1,498 @@ +#include "preflight.h" + +#include + +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; + +namespace easy_gui +{ + namespace + { + std::string lower(std::string s) + { + std::transform( + s.begin(), + s.end(), + s.begin(), + [](unsigned char c) + { + return static_cast(std::tolower(c)); + }); + return s; + } + + std::string names(const std::vector& list, size_t max_shown = 5) + { + std::string out; + for (size_t i = 0; i < list.size() && i < max_shown; i++) + out += (i ? ", " : "") + list[i]; + if (list.size() > max_shown) + out += " and " + std::to_string(list.size() - max_shown) + " more"; + return out; + } + + // Last 4 characters of the file stem, as load_data's get4index. + std::string index4(const fs::path& p) + { + const auto s = p.stem().string(); + return s.size() > 4 ? s.substr(s.size() - 4) : s; + } + + struct LazSummary + { + bool readable = false; + uint64_t points = 0; + double first_time = std::numeric_limits::max(); + double last_time = std::numeric_limits::lowest(); + std::set lidar_ids; + }; + + // Header + a sample of points from the start and the end of the file: enough to check the lidar id and the + // time range without decompressing whole files. + LazSummary summarizeLaz(const fs::path& file) + { + constexpr uint64_t kSample = 20000; + LazSummary s; + laszip_POINTER reader = nullptr; + if (laszip_create(&reader)) + return s; + + laszip_BOOL compressed = 0; + laszip_header* header = nullptr; + laszip_point* point = nullptr; + if (laszip_open_reader(reader, file.string().c_str(), &compressed) || laszip_get_header_pointer(reader, &header) || + laszip_get_point_pointer(reader, &point)) + { + laszip_destroy(reader); + return s; + } + + s.points = header->number_of_point_records ? header->number_of_point_records : header->extended_number_of_point_records; + s.readable = true; + + auto sample = [&](uint64_t begin, uint64_t end) + { + for (uint64_t i = begin; i < end; i++) + { + if (laszip_read_point(reader)) + { + s.readable = false; + return; + } + s.lidar_ids.insert(point->user_data); + if (point->gps_time != 0.0) + { + s.first_time = std::min(s.first_time, point->gps_time); + s.last_time = std::max(s.last_time, point->gps_time); + } + } + }; + + sample(0, std::min(kSample, s.points)); + if (s.readable && s.points > kSample) + { + const uint64_t tail_begin = std::max(kSample, s.points - std::min(1000, s.points)); + if (laszip_seek_point(reader, static_cast(tail_begin))) + s.readable = false; + else + sample(tail_begin, s.points); + } + + laszip_close_reader(reader); + laszip_destroy(reader); + return s; + } + + // IMU timestamps (ns in the file) of the first and last sample, in seconds. Handles the named-column format + // and the legacy headerless one, like load_imu(). + bool imuTimeRange(const fs::path& first_file, const fs::path& last_file, double& t_first, double& t_last) + { + auto split = [](const std::string& line) + { + std::vector out; + std::string token; + for (char c : line) + { + if (c == ' ' || c == ',' || c == '\t' || c == '\r') + { + if (!token.empty()) + out.push_back(token); + token.clear(); + } + else + token += c; + } + if (!token.empty()) + out.push_back(token); + return out; + }; + + auto timestampColumn = [&](const fs::path& file, size_t& column, bool& has_header) -> bool + { + std::ifstream in(file); + std::string line; + if (!std::getline(in, line)) + return false; + const auto cols = split(line); + has_header = false; + column = 0; + for (size_t i = 0; i < cols.size(); i++) + if (cols[i] == "timestamp") + { + has_header = true; + column = i; + } + return true; + }; + + auto parseTime = [&](const std::string& line, size_t column, double& t) -> bool + { + const auto cols = split(line); + if (column >= cols.size()) + return false; + try + { + t = std::stod(cols[column]) / 1e9; + } catch (...) + { + return false; + } + return t > 0.0; + }; + + size_t column = 0; + bool has_header = false; + if (!timestampColumn(first_file, column, has_header)) + return false; + + { + std::ifstream in(first_file); + std::string line; + if (has_header) + std::getline(in, line); + bool found = false; + while (!found && std::getline(in, line)) + found = parseTime(line, column, t_first); + if (!found) + return false; + } + { + size_t last_column = 0; + bool last_has_header = false; + if (!timestampColumn(last_file, last_column, last_has_header)) + return false; + std::ifstream in(last_file); + std::string line; + std::string last_good; + double t = 0.0; + bool first = true; + while (std::getline(in, line)) + { + if (first && last_has_header) + { + first = false; + continue; + } + first = false; + if (parseTime(line, last_column, t)) + t_last = t; + } + } + return t_last > t_first; + } + } // namespace + + bool isSameOrInside(const fs::path& inner, const fs::path& outer) + { + std::error_code ec; + const fs::path a = fs::weakly_canonical(fs::absolute(inner, ec), ec).lexically_normal(); + const fs::path b = fs::weakly_canonical(fs::absolute(outer, ec), ec).lexically_normal(); + auto ai = a.begin(); + for (auto bi = b.begin(); bi != b.end(); ++bi, ++ai) + { + if (bi->empty()) // trailing separator + continue; + if (ai == a.end() || *ai != *bi) + return false; + } + return true; + } + + std::string formatBytes(uint64_t bytes) + { + std::ostringstream out; + out.precision(1); + out << std::fixed << static_cast(bytes) / 1e9 << " GB"; + return out.str(); + } + + bool availableSpace(const fs::path& path, uint64_t& available, fs::path& checked) + { + std::error_code ec; + checked = fs::absolute(path, ec); + while (!checked.empty() && !fs::exists(checked, ec) && checked != checked.parent_path()) + checked = checked.parent_path(); + const auto space = fs::space(checked, ec); + if (ec) + return false; + available = space.available; + return true; + } + + bool holdsUnfinishedOutput(const fs::path& folder) + { + // NOTE: the names of OutputLayout (pipeline.h), which this library level cannot include; a test keeps the two + // in sync. + static const std::set kRunEntries = { "easy_gui.log", "work", "point_cloud.laz", "trajectory.csv", "README.txt" }; + std::error_code ec; + if (!fs::is_directory(folder, ec)) + return false; + bool any = false; + for (const auto& entry : fs::directory_iterator(folder, ec)) + { + if (!kRunEntries.contains(entry.path().filename().string())) + return false; + any = true; + } + return any && !ec; + } + + uint64_t estimateWorkBytes(uint64_t total_points) + { + return total_points * 120; + } + + uint64_t estimateResultBytes(uint64_t raw_laz_bytes) + { + // Measured on a 117 M-point forest walk (734 MB raw LAZ): session 0.82x + point cloud 1.29x (with near and gap + // points) + trajectory CSV 0.03x = 2.14x; 2.25x leaves some margin (the near-point share depends on the site). + return raw_laz_bytes * 9 / 4; + } + + RecordingFiles classifyRecordingFiles(const fs::path& recording) + { + RecordingFiles files; + std::vector entries; + std::error_code ec; + for (const auto& entry : fs::directory_iterator(recording, ec)) + if (!entry.is_directory(ec)) + entries.push_back(entry.path()); + std::sort(entries.begin(), entries.end()); + + for (const auto& p : entries) + { + const std::string ext = lower(p.extension().string()); + if (ext == ".csv") + files.csv_files.push_back(p); + else if (ext == ".laz" || ext == ".las") + files.laz_files.push_back(p); + else if (ext == ".json") + { + const std::string stem = p.stem().string(); + if (!stem.starts_with("status") && !stem.starts_with("cam0") && !stem.starts_with("cam1") && files.calibration_file.empty()) + files.calibration_file = p; + } + else if (ext == ".mjc" && files.calibration_file.empty()) + files.calibration_file = p; + else if (ext == ".sn" && files.sn_file.empty()) + files.sn_file = p; + } + files.pairs = std::min(files.laz_files.size(), files.csv_files.size()); + return files; + } + + PreflightResult runPreflight(const fs::path& recording, const fs::path& output_folder, bool resume_unfinished) + { + PreflightResult r; + auto add = [&r](Severity s, std::string m) + { + r.findings.push_back({ s, std::move(m) }); + }; + + std::error_code ec; + if (recording.empty() || !fs::is_directory(recording, ec)) + { + add(Severity::Error, "The recording folder '" + recording.string() + "' does not exist or is not a folder."); + return r; + } + + r.files = classifyRecordingFiles(recording); + const auto& f = r.files; + + if (f.laz_files.empty() || f.csv_files.empty()) + { + add(Severity::Error, + std::string("This folder does not look like a Mandeye recording: ") + + (f.laz_files.empty() ? "there are no point cloud files (lidar*.laz)" : "there are no IMU files (imu*.csv)") + + ". Choose the folder the scanner wrote, for example 'continousScanning_0012'."); + return r; + } + + // --- pairing (by position, as load_data does) + if (f.laz_files.size() != f.csv_files.size()) + { + const auto& longer = f.laz_files.size() > f.csv_files.size() ? f.laz_files : f.csv_files; + std::vector skipped; + for (size_t i = f.pairs; i < longer.size(); i++) + skipped.push_back(longer[i].filename().string()); + add(Severity::Warning, + "There are " + std::to_string(f.laz_files.size()) + " point cloud files but " + std::to_string(f.csv_files.size()) + + " IMU files. The unmatched file(s) at the end will not be processed: " + names(skipped) + + ". This is normal when the scanner stopped between two files."); + } + std::vector misaligned; + for (size_t i = 0; i < f.pairs; i++) + if (index4(f.laz_files[i]) != index4(f.csv_files[i])) + misaligned.push_back(f.laz_files[i].filename().string() + " / " + f.csv_files[i].filename().string()); + if (!misaligned.empty()) + add(Severity::Warning, + "Some file numbers do not match (" + names(misaligned, 3) + + "). Files are used in order, so a missing file in the middle of the recording can leave a gap in the result."); + + // --- point cloud files: readable, one lidar, time range + std::vector unreadable; + std::set lidar_ids; + double lidar_first = std::numeric_limits::max(); + double lidar_last = std::numeric_limits::lowest(); + for (size_t i = 0; i < f.pairs; i++) + { + const auto s = summarizeLaz(f.laz_files[i]); + if (!s.readable) + { + unreadable.push_back(f.laz_files[i].filename().string()); + continue; + } + r.total_points += s.points; + r.raw_laz_bytes += fs::file_size(f.laz_files[i], ec); + lidar_ids.insert(s.lidar_ids.begin(), s.lidar_ids.end()); + lidar_first = std::min(lidar_first, s.first_time); + lidar_last = std::max(lidar_last, s.last_time); + } + if (!unreadable.empty()) + add(Severity::Error, + "These point cloud files cannot be read (damaged, or not fully copied from the scanner): " + names(unreadable) + + ". Copy the recording again, or remove the damaged files at the end of the recording."); + if (lidar_ids.size() > 1) + { + std::string ids; + for (int id : lidar_ids) + ids += (ids.empty() ? "" : ", ") + std::to_string(id); + add(Severity::Error, + "This recording contains data from more than one lidar (ids " + ids + + "). Easy Forest HDMapping supports single-lidar recordings only; use the standard HDMapping apps for this one."); + } + else if (lidar_ids.size() == 1) + r.lidar_id = *lidar_ids.begin(); + + // --- calibration (mirrors load_data's checks, which otherwise end in std::exit) + const auto id_to_sn = MLvxCalib::GetIdToSnMapping(f.sn_file.string()); + if (!f.calibration_file.empty()) + { + const auto calibration = MLvxCalib::GetCalibrationFromFile(f.calibration_file.string()); + const std::string imu_sn = MLvxCalib::GetImuSnToUse(f.calibration_file.string()); + if (calibration.empty()) + add(Severity::Warning, + "The calibration file '" + f.calibration_file.filename().string() + "' could not be read and will be ignored."); + else if (calibration.size() > 1) + add(Severity::Error, + "The calibration file '" + f.calibration_file.filename().string() + + "' describes more than one lidar. Easy Forest HDMapping supports single-lidar recordings only."); + else + { + for (const auto& [id, sn] : id_to_sn) + if (!calibration.contains(sn)) + add(Severity::Error, + "The calibration file '" + f.calibration_file.filename().string() + + "' belongs to a different scanner (this recording was made with lidar " + sn + ")."); + if (!calibration.contains(imu_sn)) + add(Severity::Error, + "The calibration file '" + f.calibration_file.filename().string() + "' does not name a valid IMU ('" + imu_sn + + "' is not one of its sensors)."); + } + } + if (id_to_sn.size() > 1) + add(Severity::Error, + "The scanner serial-number file lists more than one lidar. Easy Forest HDMapping supports single-lidar recordings only."); + + // --- IMU vs lidar clocks + double imu_first = 0.0, imu_last = 0.0; + if (!imuTimeRange(f.csv_files.front(), f.csv_files[f.pairs - 1], imu_first, imu_last)) + add(Severity::Error, + "The IMU files could not be read (no valid timestamps in '" + f.csv_files.front().filename().string() + "')."); + else + { + r.duration_s = imu_last - imu_first; + if (lidar_last > lidar_first) + { + const double overlap = std::min(imu_last, lidar_last) - std::max(imu_first, lidar_first); + if (overlap <= 0.0) + add(Severity::Error, + "The IMU and point cloud timestamps do not overlap, so they cannot be combined. The recording is probably " + "damaged."); + } + if (r.duration_s < 20.0) + add(Severity::Warning, + "The recording is very short (" + std::to_string(static_cast(r.duration_s)) + + " s). Lidar odometry needs some movement; results may be poor."); + } + + // --- output folder + r.estimated_work_bytes = estimateWorkBytes(r.total_points); + r.estimated_result_bytes = estimateResultBytes(r.raw_laz_bytes); + r.estimated_disk_bytes = r.estimated_work_bytes + r.estimated_result_bytes; + const std::string need = "about " + formatBytes(r.estimated_disk_bytes) + " (" + formatBytes(r.estimated_work_bytes) + + " temporary during processing + " + formatBytes(r.estimated_result_bytes) + " of results)"; + std::string disk_summary; + if (output_folder.empty()) + add(Severity::Error, "Choose an output folder."); + else if (isSameOrInside(output_folder, recording)) + add(Severity::Error, + "The output folder is inside the recording folder. Choose a folder elsewhere: Easy Forest HDMapping never writes into the " + "recording " + "(it may be on the scanner's USB stick)."); + else + { + if (resume_unfinished && holdsUnfinishedOutput(output_folder)) + add(Severity::Ok, + "The output folder holds an unfinished earlier attempt. It will be replaced; its log is kept and continued."); + else if (fs::exists(output_folder, ec) && !fs::is_empty(output_folder, ec)) + add(Severity::Error, + "The output folder '" + output_folder.string() + "' already contains files. Choose an empty or new folder."); + + uint64_t available = 0; + fs::path existing; + if (!availableSpace(output_folder, available, existing)) + add(Severity::Warning, "Could not determine the free disk space at '" + existing.string() + "'."); + else if (available < r.estimated_disk_bytes) + add(Severity::Error, + "Not enough free disk space for the output folder: " + need + " needed, " + formatBytes(available) + + " free. Free some space or choose a folder on another disk."); + else if (available < r.estimated_disk_bytes + r.estimated_disk_bytes / 2) + add(Severity::Warning, "Free disk space is tight: " + need + " needed, " + formatBytes(available) + " free."); + else + disk_summary = "Disk space: " + need + " needed, " + formatBytes(available) + " free."; + } + if (!disk_summary.empty() && r.ok()) + r.findings.push_back({ Severity::Ok, disk_summary }); + + if (r.ok()) + { + std::ostringstream summary; + summary.precision(1); + summary << std::fixed << "Recording looks good: " << f.pairs << " file pairs, " << r.total_points / 1000000.0 + << " million points, " << r.duration_s / 60.0 << " minutes."; + r.findings.insert(r.findings.begin(), { Severity::Ok, summary.str() }); + } + return r; + } +} // namespace easy_gui diff --git a/apps/easy_gui/src/preflight.h b/apps/easy_gui/src/preflight.h new file mode 100644 index 00000000..aea965c0 --- /dev/null +++ b/apps/easy_gui/src/preflight.h @@ -0,0 +1,90 @@ +#pragma once + +#include +#include +#include +#include + +namespace easy_gui +{ + enum class Severity + { + Ok, + Warning, + Error + }; + + struct Finding + { + Severity severity; + std::string message; // plain language, shown as-is in the GUI + }; + + //! The recording's files, classified exactly as load_data() in lidar_odometry.cpp does it (sorted names; every + //! .laz/.las is a point cloud, every .csv an IMU file; status*/cam0*/cam1* JSON ignored; the first other .json or + //! .mjc is the calibration; the first .sn is used). load_data pairs point clouds and IMU files *by position* and + //! only loads min(#laz, #csv) pairs, so `pairs` is what step 1 will actually read. + struct RecordingFiles + { + std::vector laz_files; + std::vector csv_files; + std::filesystem::path sn_file; + std::filesystem::path calibration_file; + size_t pairs = 0; + }; + + RecordingFiles classifyRecordingFiles(const std::filesystem::path& recording); + + struct PreflightResult + { + RecordingFiles files; + std::vector findings; + uint64_t total_points = 0; // point records in the paired LAZ files (incl. no-return points) + double duration_s = 0.0; // IMU time span + int lidar_id = -1; // the single lidar id (LAZ user_data) seen in the samples + uint64_t raw_laz_bytes = 0; // size of the paired LAZ files + uint64_t estimated_work_bytes = 0; // temporary, freed after the run + uint64_t estimated_result_bytes = 0; // kept: step 1 session + exported point cloud and trajectory + uint64_t estimated_disk_bytes = 0; // work + results: what must be free when the run starts + + bool ok() const + { + for (const auto& f : findings) + if (f.severity == Severity::Error) + return false; + return true; + } + }; + + //! Checks the recording and the output folder before anything is started. Catches, with a plain message, the + //! cases where step 1 would otherwise abort() (unreadable LAZ), exit() (calibration mismatch) or silently produce + //! something incomplete (unpaired files, non-overlapping clocks). + //! `resume_unfinished` (batch mode): an output folder that holds only an unfinished earlier attempt + //! (holdsUnfinishedOutput) is accepted too; the pipeline clears it before the run. + PreflightResult runPreflight( + const std::filesystem::path& recording, const std::filesystem::path& output_folder, bool resume_unfinished = false); + + //! True if `folder` exists, is not empty and holds nothing but what an Easy GUI run writes (see OutputLayout): + //! easy_gui.log, work/, point_cloud.laz, trajectory.csv, README.txt. A finished batch result also holds the done + //! marker, so it is not "unfinished". + bool holdsUnfinishedOutput(const std::filesystem::path& folder); + + //! Free bytes on the disk that holds `path`, or of its nearest existing parent (the output folder usually does + //! not exist yet). False if it cannot be determined; `checked` is the folder that was asked. + bool availableSpace(const std::filesystem::path& path, uint64_t& available, std::filesystem::path& checked); + + //! "1.2 GB" + std::string formatBytes(uint64_t bytes); + + //! Temporary disk space of a run: step 1's caches (original + decimated + concatenated chunk points, ~45 B/point + //! each with decimation 0.01). Measured peak on a 117 M-point forest walk: ~108 B per raw point; 120 B leaves + //! some margin. + uint64_t estimateWorkBytes(uint64_t total_points); + + //! Disk space of what is kept: the step 1 session (scan_lio LAZ, 0.82x the raw LAZ size measured), the exported + //! LAZ (1.4 PF6 with extra bytes, near and gap points, 1.29x measured) and the trajectory CSV (0.03x): 2.25x. + uint64_t estimateResultBytes(uint64_t raw_laz_bytes); + + //! True if `inner` is `outer` or lies inside it (both made absolute and normalised first). + bool isSameOrInside(const std::filesystem::path& inner, const std::filesystem::path& outer); +} // namespace easy_gui diff --git a/apps/easy_gui/src/progress_parser.cpp b/apps/easy_gui/src/progress_parser.cpp new file mode 100644 index 00000000..d80b36b0 --- /dev/null +++ b/apps/easy_gui/src/progress_parser.cpp @@ -0,0 +1,209 @@ +#include "progress_parser.h" + +#include +#include + +namespace easy_gui +{ + namespace + { + struct KnownFailure + { + const char* marker; + const char* message; + }; + + // Lines that mean step 1 is going to fail (or already has), with what to tell the user. + const KnownFailure kKnownFailures[] = { + { "MISSING CALIBRATION", + "The calibration file does not match this scanner, so the recording cannot be processed with it. Remove the calibration " + "file from the recording, or use the one that belongs to this scanner." }, + { "WRONG CALIBRATION FILE", + "The calibration file belongs to a different scanner. Remove it from the recording, or use the one that belongs to this " + "scanner." }, + { "System is out of memory", + "The computer ran out of memory. Close other programs and try again, or process the recording on a computer with more " + "memory." }, + { "Loading poses went wrong", + "No trajectory could be computed from the IMU data. The IMU files are probably empty or damaged." }, + { "DLL ERROR", + "A point cloud file could not be read (damaged, or not fully copied from the scanner). Copy the recording again." }, + { "points_global spill", "Writing temporary data failed. The disk is probably full: free some space and try again." }, + { "changed or vanished since it was first read", + "A recording file changed or disappeared during processing (was the USB stick removed?). Keep the recording available " + "until processing has finished." }, + { "Lidar odometry failed", + "Lidar odometry did not succeed on this recording. Check that the scanner was moving steadily and that the recording " + "is complete; the details below show where it stopped." }, + }; + + const std::regex kFilesTotal(R"(Loading point clouds \((\d+) files\))"); + const std::regex kPointsKept(R"(^TOTAL: (\d+))"); + const std::regex kPrepare(R"(running iterations: (\d+)/(\d+))"); + const std::regex kOdometry(R"(finished optimizing .* - (\d+) / (\d+) at iteration)"); + const std::regex kSavedScan(R"(saving to: .*scan_lio_(\d+)\.laz)"); + + int toInt(const std::ssub_match& m) + { + try + { + return std::stoi(m.str()); + } catch (...) + { + return 0; + } + } + + // save_result writes one scan_lio file per 51 odometry chunks (its "counter > 50" grouping). + int expectedScans(int chunks) + { + return std::max(1, (chunks + 50) / 51); + } + } // namespace + + const char* phaseLabel(Step1Phase phase) + { + switch (phase) + { + case Step1Phase::Starting: + return "Starting"; + case Step1Phase::LoadingFiles: + return "Reading the recording"; + case Step1Phase::ImuTrajectory: + return "Computing the motion from the IMU"; + case Step1Phase::Preparing: + return "Preparing lidar odometry"; + case Step1Phase::Odometry: + return "Lidar odometry"; + case Step1Phase::Saving: + return "Saving the odometry result"; + case Step1Phase::Finished: + return "Lidar odometry finished"; + } + return ""; + } + + double Step1Progress::phaseFraction() const + { + auto ratio = [](int done, int total) + { + return total > 0 ? std::clamp(static_cast(done) / total, 0.0, 1.0) : -1.0; + }; + switch (phase) + { + case Step1Phase::LoadingFiles: + return ratio(files_done, files_total); + case Step1Phase::Preparing: + return ratio(prepare_done, prepare_total); + case Step1Phase::Odometry: + return ratio(chunks_done, chunks_total); + case Step1Phase::Saving: + return ratio(scans_saved, expectedScans(chunks_total)); + case Step1Phase::Finished: + return 1.0; + default: + return -1.0; + } + } + + double Step1Progress::overallFraction() const + { + // Share of the run time per phase, measured on a 10-minute forest walk (115 file pairs, 93 M points kept). + struct Span + { + Step1Phase phase; + double weight; + }; + static const Span kSpans[] = { + { Step1Phase::Starting, 0.0 }, { Step1Phase::LoadingFiles, 0.04 }, { Step1Phase::ImuTrajectory, 0.01 }, + { Step1Phase::Preparing, 0.05 }, { Step1Phase::Odometry, 0.85 }, { Step1Phase::Saving, 0.05 }, + }; + if (phase == Step1Phase::Finished) + return 1.0; + double done = 0.0; + for (const auto& span : kSpans) + { + if (span.phase == phase) + { + const double f = phaseFraction(); + return done + span.weight * (f < 0.0 ? 0.0 : f); + } + done += span.weight; + } + return done; + } + + void ProgressParser::advance(Step1Phase phase) + { + // Phases only move forward: a stray late line from an earlier phase must not move the bar backwards. + if (static_cast(phase) > static_cast(progress_.phase)) + progress_.phase = phase; + } + + void ProgressParser::fail(const std::string& reason, const std::string& line) + { + progress_.last_error_line = line; + if (!progress_.failed) // the first recognised cause is the real one; later lines are consequences + { + progress_.failed = true; + progress_.failure = reason; + } + } + + void ProgressParser::feed(const std::string& line) + { + for (const auto& known : kKnownFailures) + if (line.find(known.marker) != std::string::npos) + { + fail(known.message, line); + return; + } + if (line.find("[error]") != std::string::npos || line.find("ERROR") != std::string::npos) + progress_.last_error_line = line; + + std::smatch m; + if (line.find("Loading data..") != std::string::npos) + advance(Step1Phase::LoadingFiles); + else if (std::regex_search(line, m, kFilesTotal)) + { + advance(Step1Phase::LoadingFiles); + progress_.files_total = toInt(m[1]); + } + else if (line.find("counter_filtered_points:") != std::string::npos) + progress_.files_done++; // phaseFraction() clamps + else if (std::regex_search(line, m, kPointsKept)) + { + try + { + progress_.points_kept = std::stoll(m[1].str()); + } catch (...) + { + } + } + else if (line.find("start calculating trajectory") != std::string::npos) + advance(Step1Phase::ImuTrajectory); + else if (std::regex_search(line, m, kPrepare)) + { + advance(Step1Phase::Preparing); + progress_.prepare_done = toInt(m[1]); + progress_.prepare_total = toInt(m[2]); + } + else if (line.find("step_1_done") != std::string::npos) + progress_.prepare_done = progress_.prepare_total; + else if (std::regex_search(line, m, kOdometry)) + { + advance(Step1Phase::Odometry); + progress_.chunks_done = std::max(progress_.chunks_done, toInt(m[1])); + progress_.chunks_total = toInt(m[2]); + } + else if (std::regex_search(line, m, kSavedScan)) + { + advance(Step1Phase::Saving); + progress_.scans_saved = std::max(progress_.scans_saved, toInt(m[1]) + 1); + } + else if (line.find("original_points_to_save_cache_file_name") != std::string::npos) + advance(Step1Phase::Saving); + else if (line.find("remove cache:") != std::string::npos && line.find("FINISHED") != std::string::npos) + advance(Step1Phase::Finished); + } +} // namespace easy_gui diff --git a/apps/easy_gui/src/progress_parser.h b/apps/easy_gui/src/progress_parser.h new file mode 100644 index 00000000..b08c0c07 --- /dev/null +++ b/apps/easy_gui/src/progress_parser.h @@ -0,0 +1,63 @@ +#pragma once + +#include + +namespace easy_gui +{ + //! Phases of the lidar_odometry_step_1 child process, in the order they occur. + enum class Step1Phase + { + Starting, + LoadingFiles, // reading the LAZ files (load_data) + ImuTrajectory, // AHRS / initial trajectory from the IMU (calculate_trajectory) + Preparing, // cutting the trajectory into odometry chunks (compute_step_1) + Odometry, // the lidar odometry itself (compute_step_2) + Saving, // save_result + Finished + }; + + const char* phaseLabel(Step1Phase phase); + + struct Step1Progress + { + Step1Phase phase = Step1Phase::Starting; + int files_total = 0; + int files_done = 0; + int prepare_done = 0; + int prepare_total = 0; + int chunks_done = 0; + int chunks_total = 0; + int scans_saved = 0; + long long points_kept = 0; // "TOTAL: n" after loading: points left after the load filter + + bool failed = false; + std::string failure; // plain-language reason, if one was recognised + std::string last_error_line; // last raw line that looked like an error, for the details view + + //! Fraction 0..1 within the current phase, or -1 if the phase gives no measurable progress. + double phaseFraction() const; + //! Rough fraction 0..1 of the whole step 1 run (odometry dominates the run time). + double overallFraction() const; + }; + + //! Turns lidar_odometry_step_1's console output into progress. + //! + //! NOTE: this parses free-form log text that step 1 does not treat as an interface. It only uses a few stable + //! markers (see the unit test, which feeds an excerpt of a real log), never fails on unknown lines, and the GUI + //! falls back to an indeterminate bar if a phase never reports numbers. Parallel file loading can interleave + //! lines, so per-file progress counts markers instead of relying on line order. + class ProgressParser + { + public: + void feed(const std::string& line); + const Step1Progress& progress() const + { + return progress_; + } + + private: + void advance(Step1Phase phase); + void fail(const std::string& reason, const std::string& line); + Step1Progress progress_; + }; +} // namespace easy_gui diff --git a/apps/easy_gui/src/settings.h b/apps/easy_gui/src/settings.h new file mode 100644 index 00000000..01a64c6d --- /dev/null +++ b/apps/easy_gui/src/settings.h @@ -0,0 +1,28 @@ +#pragma once + +#include + +namespace easy_gui +{ + //! User-adjustable settings. Everything else comes from the forestry preset. + struct Settings + { + //! Near points (horizontal range below the 1.0 m load filter) are exported only from this 3D range outwards. + //! 0.4 m: on measured raw data the operator's body/handle shows up only as small blobs inside 0.4 m. + double near_min_range_m = 0.4; + static constexpr double kNearMinRangeLowest = 0.1; + static constexpr double kNearMinRangeHighest = 0.95; // must stay below the 1.0 m load filter + + //! Export option (default off while it is being evaluated): place all points from the raw files at their real + //! time instead of using lidar odometry's points (ExportRequest::place_by_raw_time). + bool place_by_raw_time = false; + + //! Keep the step 1 staging/preview folders after a successful run (for debugging). + bool keep_work_files = false; + + void clamp() + { + near_min_range_m = std::clamp(near_min_range_m, kNearMinRangeLowest, kNearMinRangeHighest); + } + }; +} // namespace easy_gui diff --git a/apps/easy_gui/src/sleep_blocker.cpp b/apps/easy_gui/src/sleep_blocker.cpp new file mode 100644 index 00000000..0c2d6aed --- /dev/null +++ b/apps/easy_gui/src/sleep_blocker.cpp @@ -0,0 +1,49 @@ +#include "sleep_blocker.h" + +#if defined(__APPLE__) +#include +#include +#elif defined(_WIN32) +#include +#endif + +namespace easy_gui +{ +#if defined(__APPLE__) + SleepBlocker::SleepBlocker(const std::string& reason) + { + CFStringRef name = CFStringCreateWithCString(kCFAllocatorDefault, reason.c_str(), kCFStringEncodingUTF8); + IOPMAssertionID id = 0; + active_ = name && + IOPMAssertionCreateWithName(kIOPMAssertionTypePreventUserIdleSystemSleep, kIOPMAssertionLevelOn, name, &id) == kIOReturnSuccess; + if (name) + CFRelease(name); + assertion_id_ = id; + } + + SleepBlocker::~SleepBlocker() + { + if (active_) + IOPMAssertionRelease(assertion_id_); + } +#elif defined(_WIN32) + SleepBlocker::SleepBlocker(const std::string&) + { + active_ = SetThreadExecutionState(ES_CONTINUOUS | ES_SYSTEM_REQUIRED) != 0; + } + + SleepBlocker::~SleepBlocker() + { + if (active_) + SetThreadExecutionState(ES_CONTINUOUS); + } +#else + SleepBlocker::SleepBlocker(const std::string&) + { + } + + SleepBlocker::~SleepBlocker() + { + } +#endif +} // namespace easy_gui diff --git a/apps/easy_gui/src/sleep_blocker.h b/apps/easy_gui/src/sleep_blocker.h new file mode 100644 index 00000000..cbe49509 --- /dev/null +++ b/apps/easy_gui/src/sleep_blocker.h @@ -0,0 +1,31 @@ +#pragma once + +#include + +namespace easy_gui +{ + //! Keeps the computer from going to sleep on its own while it exists (the display may still turn off). A batch can + //! run for hours unattended, and idle sleep would suspend lidar odometry halfway. + //! - macOS: an IOKit power assertion (kIOPMAssertionTypePreventUserIdleSystemSleep), visible in `pmset -g assertions`. + //! - Windows: SetThreadExecutionState(ES_CONTINUOUS | ES_SYSTEM_REQUIRED). It applies to the calling thread, so a + //! SleepBlocker must be destroyed on the thread that created it. + //! - elsewhere: does nothing. + class SleepBlocker + { + public: + explicit SleepBlocker(const std::string& reason); + ~SleepBlocker(); + SleepBlocker(const SleepBlocker&) = delete; + SleepBlocker& operator=(const SleepBlocker&) = delete; + + //! Whether the system accepted the request. + bool active() const + { + return active_; + } + + private: + bool active_ = false; + unsigned int assertion_id_ = 0; // macOS IOPMAssertionID + }; +} // namespace easy_gui diff --git a/apps/easy_gui/src/staging.cpp b/apps/easy_gui/src/staging.cpp new file mode 100644 index 00000000..84017c44 --- /dev/null +++ b/apps/easy_gui/src/staging.cpp @@ -0,0 +1,69 @@ +#include "staging.h" + +namespace fs = std::filesystem; + +namespace easy_gui +{ + bool createStagingFolder(const RecordingFiles& files, const fs::path& staging, std::string& method, std::string& error) + { + std::error_code ec; + if (fs::exists(staging, ec)) + { + error = "The work folder '" + staging.string() + "' already exists."; + return false; + } + if (!fs::create_directories(staging, ec)) + { + error = "Could not create the work folder '" + staging.string() + "' (" + ec.message() + ")."; + return false; + } + + std::vector sources; + for (size_t i = 0; i < files.pairs; i++) + { + sources.push_back(files.laz_files[i]); + sources.push_back(files.csv_files[i]); + } + if (!files.sn_file.empty()) + sources.push_back(files.sn_file); + if (!files.calibration_file.empty()) + sources.push_back(files.calibration_file); + + enum class Method + { + Symlink, + Hardlink, + Copy + }; + Method m = Method::Symlink; + + for (const auto& source : sources) + { + const fs::path absolute_source = fs::absolute(source, ec); + const fs::path target = staging / source.filename(); + for (;;) + { + ec.clear(); + if (m == Method::Symlink) + fs::create_symlink(absolute_source, target, ec); + else if (m == Method::Hardlink) + fs::create_hard_link(absolute_source, target, ec); + else + fs::copy_file(absolute_source, target, ec); + if (!ec) + break; + if (m == Method::Copy) + { + error = "Could not prepare '" + source.filename().string() + "' for processing (" + ec.message() + ")."; + return false; + } + // Fall back for this and all following files; anything half-created is overwritten by the next try. + fs::remove(target, ec); + m = (m == Method::Symlink) ? Method::Hardlink : Method::Copy; + } + } + + method = (m == Method::Symlink) ? "symbolic links" : (m == Method::Hardlink) ? "hard links" : "copies"; + return true; + } +} // namespace easy_gui diff --git a/apps/easy_gui/src/staging.h b/apps/easy_gui/src/staging.h new file mode 100644 index 00000000..b2a022db --- /dev/null +++ b/apps/easy_gui/src/staging.h @@ -0,0 +1,20 @@ +#pragma once + +#include "preflight.h" + +#include +#include + +namespace easy_gui +{ + //! Fills `staging` (created, must not exist yet) with links to the files step 1 will read: the paired point cloud + //! and IMU files, the .sn file and the calibration file. + //! + //! Why: load_data() writes its cache/ and preview/ folders next to its input files, and the recording may be on + //! the scanner's USB stick. Step 1 runs on the staging folder instead, so nothing is ever written into the + //! recording. Only the paired files are linked, which is exactly what load_data would load anyway. + //! + //! Tries a symbolic link per file, then a hard link, then a copy (Windows without developer mode cannot create + //! symlinks; hard links only work on the same volume). `method` reports which one was used. + bool createStagingFolder(const RecordingFiles& files, const std::filesystem::path& staging, std::string& method, std::string& error); +} // namespace easy_gui diff --git a/apps/easy_gui/src/version.h b/apps/easy_gui/src/version.h new file mode 100644 index 00000000..1f05e460 --- /dev/null +++ b/apps/easy_gui/src/version.h @@ -0,0 +1,18 @@ +#pragma once + +#include + +#include + +namespace easy_gui +{ + //! Easy Forest HDMapping's own release version (EASY_FOREST_VERSION in apps/easy_gui/CMakeLists.txt, matching the + //! easy-forest-v* release tags). HDMapping's version is a different number, so user-facing text names both. + inline constexpr const char* kEasyForestVersion = EASY_FOREST_VERSION; + + //! "Easy Forest HDMapping 0.2.0 (HDMapping 0.105.0)": window title, README.txt, easy_gui.log. + inline std::string versionString() + { + return std::string("Easy Forest HDMapping ") + kEasyForestVersion + " (HDMapping " + HDMAPPING_CONFIGURE_PROJECT_VERSION + ")"; + } +} // namespace easy_gui diff --git a/apps/easy_gui/tests/CMakeLists.txt b/apps/easy_gui/tests/CMakeLists.txt new file mode 100644 index 00000000..82eb523e --- /dev/null +++ b/apps/easy_gui/tests/CMakeLists.txt @@ -0,0 +1,29 @@ +cmake_minimum_required(VERSION 4.0.0) + +project(easy_gui_tests) + +# Unit tests for apps/easy_gui's easy_gui_core (everything but the window). +# Doctest, like the other suites. The pre-flight and staging tests write small +# synthetic Mandeye recordings to the temp directory (the export tests also a +# synthetic step 1 session in step 1's file formats); the progress parser test +# replays an excerpt of a real lidar_odometry_step_1 log. +add_executable(easy_gui_tests + test_main.cpp + test_batch.cpp + test_child_process.cpp + test_export.cpp + test_pipeline_integration.cpp + test_preflight.cpp + test_progress_parser.cpp + test_staging_and_preset.cpp +) + +target_include_directories(easy_gui_tests PRIVATE ${THIRDPARTY_DIRECTORY}/doctest) +target_link_libraries(easy_gui_tests PRIVATE easy_gui_core) + +if (MSVC) + target_compile_options(easy_gui_tests PRIVATE /bigobj) +endif() + +include(CTest) +add_test(NAME easy_gui_tests COMMAND easy_gui_tests) diff --git a/apps/easy_gui/tests/test_batch.cpp b/apps/easy_gui/tests/test_batch.cpp new file mode 100644 index 00000000..3938210d --- /dev/null +++ b/apps/easy_gui/tests/test_batch.cpp @@ -0,0 +1,555 @@ +#include + +#include "batch.h" +#include "test_helpers.h" + +#include +#include +#include + +using namespace easy_gui; +using easy_gui_test::TempDir; +namespace fs = std::filesystem; + +namespace +{ + std::vector readLines(const fs::path& file) + { + std::ifstream in(file); + std::vector lines; + std::string line; + while (std::getline(in, line)) + lines.push_back(line); + return lines; + } + + const DiscoveredFolder* findFolder(const std::vector& folders, const std::string& name) + { + for (const auto& f : folders) + if (f.name == name) + return &f; + return nullptr; + } + + //! Stands in for the Pipeline: recordings whose folder name contains "fail" fail, the others succeed and leave + //! a result file. With `block` set, a job stays busy until release() or cancel(). + struct FakeJobs + { + std::mutex mutex; + std::condition_variable cv; + std::vector started; + bool block = false; + bool released = false; + + void release() + { + std::lock_guard lock(mutex); + released = true; + cv.notify_all(); + } + + size_t startedCount() + { + std::lock_guard lock(mutex); + return started.size(); + } + + RecordingJobFactory factory() + { + return [this] + { + return std::make_unique(*this); + }; + } + + class Job : public RecordingJob + { + public: + explicit Job(FakeJobs& owner) + : owner_(owner) + { + } + ~Job() override + { + if (thread_.joinable()) + thread_.join(); + } + bool start(const PipelineRequest& request, std::string&) override + { + { + std::lock_guard lock(owner_.mutex); + owner_.started.push_back(request); + } + status_.state = PipelineState::Running; + status_.stage = PipelineStage::Odometry; + busy_ = true; + thread_ = std::thread( + [this, request] + { + bool cancelled = false; + { + std::unique_lock lock(owner_.mutex); + owner_.cv.wait( + lock, + [&] + { + return !owner_.block || owner_.released || cancel_; + }); + cancelled = cancel_; + } + std::lock_guard lock(status_mutex_); + if (cancelled) + { + status_.state = PipelineState::Cancelled; + status_.message = "Processing was cancelled."; + } + else if (request.recording.filename().string().find("fail") != std::string::npos) + { + status_.state = PipelineState::Failed; + status_.message = "Lidar odometry stopped unexpectedly (exit code 3)."; + } + else + { + fs::create_directories(request.output_folder); + easy_gui_test::writeText(request.output_folder / "point_cloud.laz", "fake"); + status_.state = PipelineState::Succeeded; + status_.export_stats.odometry_points = 1000; + status_.export_stats.near_points = 20; + status_.export_stats.gap_points = 3; + status_.elapsed_s = 12.0; + } + busy_ = false; + }); + return true; + } + bool busy() const override + { + return busy_; + } + PipelineStatus status() const override + { + std::lock_guard lock(status_mutex_); + return status_; + } + void cancel() override + { + std::lock_guard lock(owner_.mutex); + cancel_ = true; + owner_.cv.notify_all(); + } + + private: + FakeJobs& owner_; + std::thread thread_; + std::atomic busy_{ false }; + bool cancel_ = false; // guarded by owner_.mutex + mutable std::mutex status_mutex_; + PipelineStatus status_; + }; + }; + + BatchRequest prepareBatch(const fs::path& parent, const fs::path& output_root) + { + return planBatch(parent, output_root).request; + } + + BatchStatus runToEnd(BatchRunner& runner, const BatchRequest& request) + { + std::string error; + REQUIRE(runner.start(request, error)); + runner.wait(); + return runner.status(); + } + + BatchStatus waitFor(const BatchRunner& runner, bool (*done)(const BatchStatus&)) + { + for (int i = 0; i < 200; i++) + { + const BatchStatus s = runner.status(); + if (done(s)) + return s; + std::this_thread::sleep_for(std::chrono::milliseconds(25)); + } + return runner.status(); + } +} // namespace + +TEST_CASE("discoverRecordings finds recordings one level down and explains the other folders") +{ + TempDir tmp; + const fs::path parent = tmp.path / "field_day"; + easy_gui_test::writeRecording(parent / "plot_b"); + easy_gui_test::writeRecording(parent / "plot_a"); + fs::create_directories(parent / "notes"); + easy_gui_test::writeText(parent / "notes" / "todo.txt", "x"); + easy_gui_test::writeRecording(parent / "day2" / "plot_c"); // nested two levels: not searched + fs::create_directories(parent / "only_imu"); + easy_gui_test::writeImuCsv(parent / "only_imu" / "imu0000.csv", 10, 1783328629.0); + easy_gui_test::writeRecording(parent / ".Trashes"); // hidden: ignored even though it looks like a recording + easy_gui_test::writeText(parent / "readme.txt", "a file, not a folder"); + + const auto found = discoverRecordings(parent); + REQUIRE(found.size() == 5); + CHECK(found[0].name == "day2"); // sorted by name + CHECK_FALSE(found[0].is_recording); + CHECK(found[0].skip_reason.find("one folder level down") != std::string::npos); + CHECK(findFolder(found, "notes")->skip_reason.find("No point cloud files") != std::string::npos); + CHECK(findFolder(found, "only_imu")->skip_reason == "No point cloud files (lidar*.laz)."); + CHECK(findFolder(found, "plot_a")->is_recording); + CHECK(findFolder(found, "plot_b")->is_recording); + CHECK(findFolder(found, ".Trashes") == nullptr); + CHECK(findFolder(found, "plot_c") == nullptr); + + CHECK(discoverRecordings(tmp.path / "missing").empty()); +} + +TEST_CASE("discoverRecordings accepts the parent itself as a recording") +{ + TempDir tmp; + const fs::path parent = tmp.path / "continousScanning_0012"; + easy_gui_test::writeRecording(parent); + SUBCASE("alone") + { + const auto found = discoverRecordings(parent); + REQUIRE(found.size() == 1); + CHECK(found[0].is_recording); + CHECK(found[0].name == "continousScanning_0012"); + CHECK(found[0].path == parent); + } + SUBCASE("with a trailing separator and a recording inside it, listed first") + { + easy_gui_test::writeRecording(parent / "continousScanning_0012"); + const auto found = discoverRecordings(parent.string() + "/"); + REQUIRE(found.size() == 2); + CHECK(found[0].name == "continousScanning_0012"); + CHECK(found[1].name == "continousScanning_0012"); + CHECK(found[1].path.filename() == "continousScanning_0012"); + + // The same name twice: the default output names differ. + const auto names = defaultOutputNames(found, tmp.path / "out"); + CHECK(names == std::vector{ "continousScanning_0012", "continousScanning_0012_2" }); + } +} + +TEST_CASE("discoverRecordings recognises Easy Forest HDMapping results inside the parent") +{ + TempDir tmp; + easy_gui_test::writeRecording(tmp.path / "rec"); + fs::create_directories(tmp.path / "results"); + easy_gui_test::writeText(tmp.path / "results" / "batch_summary.csv", "x"); + const auto found = discoverRecordings(tmp.path); + CHECK(findFolder(found, "results")->skip_reason == "Easy Forest HDMapping results, not a recording."); +} + +TEST_CASE("output names: validation and collisions") +{ + CHECK(outputNameProblem("plot_12").empty()); + CHECK(outputNameProblem("Plot 12 (north)").empty()); + CHECK_FALSE(outputNameProblem("").empty()); + CHECK_FALSE(outputNameProblem("a/b").empty()); + CHECK_FALSE(outputNameProblem("a\\b").empty()); + CHECK_FALSE(outputNameProblem("a:b").empty()); + CHECK_FALSE(outputNameProblem("..").empty()); + CHECK_FALSE(outputNameProblem("plot.").empty()); + CHECK_FALSE(outputNameProblem("plot ").empty()); + CHECK_FALSE(outputNameProblem("CON").empty()); + CHECK_FALSE(outputNameProblem("com1.txt").empty()); + CHECK(outputNameProblem("console").empty()); + CHECK(sameOutputName("Plot_A", "plot_a")); + CHECK_FALSE(sameOutputName("plot_a", "plot_b")); + + TempDir tmp; + easy_gui_test::writeRecording(tmp.path / "x1"); + easy_gui_test::writeRecording(tmp.path / "x2", 1783329629.0); + easy_gui_test::writeRecording(tmp.path / "x3", 1783330629.0); + const fs::path out = tmp.path / "out"; + + SUBCASE("names that differ only in case get a suffix; invalid characters are replaced") + { + const std::vector recordings = { { tmp.path / "x1", "Plot", true, {} }, + { tmp.path / "x2", "plot", true, {} }, + { tmp.path / "x3", "a:b.", true, {} } }; + CHECK(defaultOutputNames(recordings, out) == std::vector{ "Plot", "plot_2", "a_b" }); + } + SUBCASE("a folder holding something else is passed over; this recording's result and unfinished attempts are reused") + { + fs::create_directories(out / "x1"); + easy_gui_test::writeText(out / "x1" / "my_notes.txt", "not ours"); + fs::create_directories(out / "x2"); + easy_gui_test::writeText(out / "x2" / "easy_gui.log", "unfinished attempt"); + fs::create_directories(out / "x3"); + std::string error; + DoneMarker marker; + marker.fingerprint = recordingFingerprint(classifyRecordingFiles(tmp.path / "x3")); + REQUIRE(writeDoneMarker(out / "x3", marker, error)); + const std::vector recordings = { { tmp.path / "x1", "x1", true, {} }, + { tmp.path / "x2", "x2", true, {} }, + { tmp.path / "x3", "x3", true, {} } }; + CHECK(defaultOutputNames(recordings, out) == std::vector{ "x1_2", "x2", "x3" }); + + // ... but another recording's finished result is not reused. + const std::vector other = { { tmp.path / "x1", "x3", true, {} } }; + CHECK(defaultOutputNames(other, out) == std::vector{ "x3_2" }); + } +} + +TEST_CASE("done markers round-trip and identify the recording by its files, not its path") +{ + TempDir tmp; + easy_gui_test::writeRecording(tmp.path / "rec"); + const auto files = classifyRecordingFiles(tmp.path / "rec"); + const std::string fingerprint = recordingFingerprint(files); + CHECK(fingerprint.find("2 file pairs") == 0); + + fs::create_directories(tmp.path / "out"); + DoneMarker m{ "/Volumes/USB/rec", fingerprint, "2026-10-18 10:00:00", 1234.0, 4000, 3900 }; + std::string error; + REQUIRE(writeDoneMarker(tmp.path / "out", m, error)); + const auto r = readDoneMarker(tmp.path / "out"); + REQUIRE(r); + CHECK(r->recording == m.recording); + CHECK(r->fingerprint == fingerprint); + CHECK(r->finished == m.finished); + CHECK(r->duration_s == 1234.0); + CHECK(r->raw_points == 4000); + CHECK(r->exported_points == 3900); + CHECK_FALSE(readDoneMarker(tmp.path / "nowhere")); + + // The same files at another path (copied off the USB stick) are the same recording. + fs::copy(tmp.path / "rec", tmp.path / "copy"); + CHECK(checkBatchRecording(tmp.path / "copy", tmp.path / "out").done.has_value()); + // A different recording is not. + easy_gui_test::writeRecording(tmp.path / "other"); + easy_gui_test::writeRawLaz(tmp.path / "other" / "lidar0001.laz", 2100, 1783328629.0 + 30.0); + CHECK_FALSE(checkBatchRecording(tmp.path / "other", tmp.path / "out").done.has_value()); +} + +TEST_CASE("the batch disk check counts the largest temporary need once and every result") +{ + PreflightResult a, b, c; + a.estimated_work_bytes = 10'000'000'000; + a.estimated_result_bytes = 2'000'000'000; + b.estimated_work_bytes = 14'000'000'000; + b.estimated_result_bytes = 3'000'000'000; + c.estimated_work_bytes = 6'000'000'000; + c.estimated_result_bytes = 1'000'000'000; + const BatchDiskNeed need = batchDiskNeed({ &a, &b, &c }); + CHECK(need.work_bytes == 14'000'000'000); + CHECK(need.result_bytes == 6'000'000'000); + CHECK(need.total() == 20'000'000'000); + CHECK(batchDiskNeed({}).total() == 0); + + CHECK(batchDiskFinding(need, true, 19'000'000'000).severity == Severity::Error); + CHECK(batchDiskFinding(need, true, 25'000'000'000).severity == Severity::Warning); + const Finding ok = batchDiskFinding(need, true, 40'000'000'000); + CHECK(ok.severity == Severity::Ok); + CHECK(ok.message.find("14.0 GB temporary for the largest recording + 6.0 GB of results") != std::string::npos); + CHECK(batchDiskFinding(need, false, 0).severity == Severity::Warning); +} + +TEST_CASE("the batch output folder must not be inside a recording") +{ + TempDir tmp; + easy_gui_test::writeRecording(tmp.path / "parent"); + easy_gui_test::writeRecording(tmp.path / "parent" / "sub"); + const auto found = discoverRecordings(tmp.path / "parent"); + CHECK_FALSE(batchOutputProblem(tmp.path / "parent" / "results", found).empty()); // the parent is a recording + CHECK_FALSE(batchOutputProblem(tmp.path / "parent" / "sub" / "out", found).empty()); + CHECK(batchOutputProblem(tmp.path / "results", found).empty()); + CHECK_FALSE(batchOutputProblem({}, found).empty()); + + // A plain container parent may hold the output folder. + easy_gui_test::writeRecording(tmp.path / "container" / "rec"); + CHECK(batchOutputProblem(tmp.path / "container" / "results", discoverRecordings(tmp.path / "container")).empty()); +} + +TEST_CASE("a failed recording does not stop the batch; the summary and the log record every folder") +{ + TempDir tmp; + const fs::path parent = tmp.path / "parent"; + easy_gui_test::writeRecording(parent / "a_plot"); + easy_gui_test::writeRecording(parent / "b_fail", 1783329629.0); + easy_gui_test::writeRecording(parent / "c_plot, north", 1783330629.0); // comma: quoted in the CSV + fs::create_directories(parent / "d_damaged"); + easy_gui_test::writeText(parent / "d_damaged" / "lidar0000.laz", "not a LAZ file"); + easy_gui_test::writeImuCsv(parent / "d_damaged" / "imu0000.csv", 6000, 1783328629.0); + fs::create_directories(parent / "e_notes"); + const fs::path out = tmp.path / "out"; + + BatchRequest request = prepareBatch(parent, out); + REQUIRE(request.items.size() == 4); + CHECK_FALSE(request.items[3].include); // d_damaged fails the check and is not selected + CHECK(request.items[3].exclude_reason.find("cannot be read") != std::string::npos); + REQUIRE(request.skipped.size() == 1); + + FakeJobs jobs; + BatchRunner runner(jobs.factory()); + const BatchStatus s = runToEnd(runner, request); + CHECK_FALSE(s.running); + CHECK(s.items[0].state == BatchItemState::Succeeded); + CHECK(s.items[1].state == BatchItemState::Failed); + CHECK(s.items[1].error == "Lidar odometry stopped unexpectedly (exit code 3)."); + CHECK(s.items[2].state == BatchItemState::Succeeded); + CHECK(s.items[3].state == BatchItemState::Excluded); + CHECK(s.items[0].exported_points == 1023); + CHECK(s.items[0].raw_points == 4000); + CHECK(s.fraction == doctest::Approx(1.0)); + REQUIRE(jobs.started.size() == 3); + CHECK(jobs.started[0].output_folder == out / "a_plot"); + CHECK(jobs.started[0].resume_unfinished); + + CHECK(readDoneMarker(out / "a_plot")); + CHECK_FALSE(readDoneMarker(out / "b_fail")); + CHECK(readDoneMarker(out / "c_plot, north")); + + const auto csv = readLines(out / BatchRunner::kSummaryFileName); + REQUIRE(csv.size() == 6); + CHECK(csv[0] == "recording,output_name,status,duration_s,raw_points,exported_points,output_path,error"); + CHECK(csv[1].find(",a_plot,succeeded,") != std::string::npos); + CHECK(csv[1].find(",4000,1023,") != std::string::npos); + CHECK(csv[2].find(",b_fail,failed,") != std::string::npos); + CHECK(csv[2].find("exit code 3") != std::string::npos); + CHECK(csv[3].find(",\"c_plot, north\",succeeded,") != std::string::npos); + CHECK(csv[4].find(",d_damaged,excluded,") != std::string::npos); + CHECK(csv[5].find(",,skipped,,,,,") != std::string::npos); + CHECK_FALSE(fs::exists(out / "batch_summary.csv.tmp")); + + const auto log = readLines(out / BatchRunner::kLogFileName); + auto logHas = [&](const std::string& text) + { + return std::any_of( + log.begin(), + log.end(), + [&](const std::string& l) + { + return l.find(text) != std::string::npos; + }); + }; + CHECK(logHas("batch started")); + CHECK(logHas("recording b_fail: FAILED")); + CHECK(logHas("skipped: e_notes")); + CHECK(logHas("excluded: d_damaged")); + CHECK(logHas("1 failed")); +} + +TEST_CASE("restarting the same batch skips finished recordings and retries the others") +{ + TempDir tmp; + const fs::path parent = tmp.path / "parent"; + easy_gui_test::writeRecording(parent / "a_plot"); + easy_gui_test::writeRecording(parent / "b_fail", 1783329629.0); + easy_gui_test::writeRecording(parent / "c_plot", 1783330629.0); + const fs::path out = tmp.path / "out"; + { + FakeJobs jobs; + BatchRunner runner(jobs.factory()); + runToEnd(runner, prepareBatch(parent, out)); + REQUIRE(jobs.started.size() == 3); + } + // b_fail left an unfinished attempt (a log and a half-written point cloud); it is retried, not refused. + fs::create_directories(out / "b_fail"); + easy_gui_test::writeText(out / "b_fail" / "easy_gui.log", "first attempt\n"); + easy_gui_test::writeText(out / "b_fail" / "point_cloud.laz", "partial"); + + const BatchRequest again = prepareBatch(parent, out); + REQUIRE(again.items.size() == 3); + CHECK(again.items[0].output_name == "a_plot"); // same names as before, so the markers are found + CHECK(again.items[1].include); + + FakeJobs jobs; + BatchRunner runner(jobs.factory()); + const BatchStatus s = runToEnd(runner, again); + CHECK(s.items[0].state == BatchItemState::AlreadyDone); + CHECK(s.items[0].exported_points == 1023); // from the marker + CHECK(s.items[1].state == BatchItemState::Failed); + CHECK(s.items[2].state == BatchItemState::AlreadyDone); + REQUIRE(jobs.started.size() == 1); + CHECK(jobs.started[0].recording == parent / "b_fail"); + const auto csv = readLines(out / BatchRunner::kSummaryFileName); + CHECK(csv[1].find(",a_plot,already_done,") != std::string::npos); +} + +TEST_CASE("a recording whose output folder holds another recording's result fails and the batch continues") +{ + TempDir tmp; + const fs::path parent = tmp.path / "parent"; + easy_gui_test::writeRecording(parent / "a_plot"); + easy_gui_test::writeRecording(parent / "b_plot", 1783329629.0); + const fs::path out = tmp.path / "out"; + BatchRequest request = prepareBatch(parent, out); + // Simulates a stale table: the folder was filled after the check. + fs::create_directories(out / "a_plot"); + easy_gui_test::writeText(out / "a_plot" / "my_notes.txt", "not ours"); + + FakeJobs jobs; + BatchRunner runner(jobs.factory()); + const BatchStatus s = runToEnd(runner, request); + CHECK(s.items[0].state == BatchItemState::Failed); + CHECK(s.items[0].error.find("already contains files") != std::string::npos); + CHECK(s.items[1].state == BatchItemState::Succeeded); + CHECK(jobs.startedCount() == 1); +} + +TEST_CASE("stop after current lets the running recording finish; stop now cancels it") +{ + TempDir tmp; + const fs::path parent = tmp.path / "parent"; + easy_gui_test::writeRecording(parent / "a_plot"); + easy_gui_test::writeRecording(parent / "b_plot", 1783329629.0); + easy_gui_test::writeRecording(parent / "c_plot", 1783330629.0); + const fs::path out = tmp.path / "out"; + + FakeJobs jobs; + jobs.block = true; + BatchRunner runner(jobs.factory()); + std::string error; + REQUIRE(runner.start(prepareBatch(parent, out), error)); + const BatchStatus running = waitFor( + runner, + [](const BatchStatus& s) + { + return s.current == 0 && s.current_status.state == PipelineState::Running; + }); + REQUIRE(running.current == 0); + CHECK(runner.busy()); + + SUBCASE("stop after current") + { + runner.stopAfterCurrent(); + CHECK(runner.status().stop_after_current); + jobs.release(); + runner.wait(); + const BatchStatus s = runner.status(); + CHECK(s.items[0].state == BatchItemState::Succeeded); + CHECK(s.items[1].state == BatchItemState::NotRun); + CHECK(s.items[2].state == BatchItemState::NotRun); + CHECK(readDoneMarker(out / "a_plot")); + } + SUBCASE("stop now") + { + runner.stopNow(); + runner.wait(); + const BatchStatus s = runner.status(); + CHECK(s.items[0].state == BatchItemState::Cancelled); + CHECK(s.items[1].state == BatchItemState::NotRun); + CHECK(s.items[2].state == BatchItemState::NotRun); + CHECK_FALSE(readDoneMarker(out / "a_plot")); + const auto csv = readLines(out / BatchRunner::kSummaryFileName); + CHECK(csv[1].find(",a_plot,cancelled,") != std::string::npos); + CHECK(csv[2].find(",b_plot,not_run,") != std::string::npos); + } + CHECK(jobs.startedCount() == 1); +} + +TEST_CASE("pipelineFraction weights odometry and export") +{ + PipelineStatus s; + s.state = PipelineState::Running; + s.stage = PipelineStage::Checking; + CHECK(pipelineFraction(s) == 0.0); + s.stage = PipelineStage::Export; + s.export_fraction = 0.5; + CHECK(pipelineFraction(s) == doctest::Approx(0.95)); + s.state = PipelineState::Succeeded; + CHECK(pipelineFraction(s) == 1.0); +} diff --git a/apps/easy_gui/tests/test_child_process.cpp b/apps/easy_gui/tests/test_child_process.cpp new file mode 100644 index 00000000..4b79f50f --- /dev/null +++ b/apps/easy_gui/tests/test_child_process.cpp @@ -0,0 +1,179 @@ +#include + +#include "child_process.h" + +#include +#include +#include +#include + +using easy_gui::ChildProcess; + +namespace +{ + std::vector readAll(ChildProcess& child) + { + std::vector lines; + std::string line; + while (child.readLine(line)) + lines.push_back(line); + return lines; + } +} // namespace + +TEST_CASE("arguments reach the child unchanged (spaces, quotes, backslashes, empty)") +{ + // easy_gui_tests itself is the child (--echo-args, see test_main.cpp). On Windows this checks the command line + // quoting, which output folders like "Documents\Easy Forest HDMapping\..." depend on. +#ifdef _WIN32 + const auto self = easy_gui::currentExecutableDirectory() / "easy_gui_tests.exe"; +#else + const auto self = easy_gui::currentExecutableDirectory() / "easy_gui_tests"; +#endif + const std::vector args = { "--echo-args", "plain", "with space", "", "quote\"inside", + "C:\\dir with space\\", "trailing\\", "a\\\\\"b" }; + ChildProcess child; + std::string error; + REQUIRE_MESSAGE(child.start(self, args, error), error); + const auto lines = readAll(child); + CHECK(child.wait() == 0); + REQUIRE(lines.size() == args.size() - 1); + for (size_t i = 1; i < args.size(); i++) + CHECK(lines[i - 1] == "[" + args[i] + "]"); +} + +#ifndef _WIN32 // uses /bin/sh + +TEST_CASE("stdout and stderr are merged line by line and the exit code is returned") +{ + ChildProcess child; + std::string error; + REQUIRE(child.start("/bin/sh", { "-c", "echo one; echo two 1>&2; printf 'three\\r\\nfour'; exit 3" }, error)); + std::vector lines; + std::string line; + while (child.readLine(line)) + lines.push_back(line); + CHECK(child.wait() == 3); + REQUIRE(lines.size() == 4); + CHECK(lines[0] == "one"); + CHECK(lines[1] == "two"); + CHECK(lines[2] == "three"); // '\r' removed + CHECK(lines[3] == "four"); // last line without a newline +} + +TEST_CASE("stdin is closed, so a program waiting for input does not hang") +{ + // load_data does std::cin.get() before std::exit on a calibration mismatch. + ChildProcess child; + std::string error; + REQUIRE(child.start("/bin/sh", { "-c", "read x; echo got-eof" }, error)); + std::string line; + REQUIRE(child.readLine(line)); + CHECK(line == "got-eof"); + CHECK(child.wait() != -1); +} + +TEST_CASE("kill stops the child and its process group promptly") +{ + ChildProcess child; + std::string error; + // The background sleep is a grandchild: it must die too, or it would hold the pipe open. + REQUIRE(child.start("/bin/sh", { "-c", "sleep 30 & echo started; sleep 30" }, error)); + std::string line; + REQUIRE(child.readLine(line)); + CHECK(line == "started"); + + const auto t0 = std::chrono::steady_clock::now(); + std::thread killer( + [&] + { + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + child.kill(); + }); + while (child.readLine(line)) + { + } + killer.join(); + CHECK(child.wait() == -1); + CHECK(child.wasKilled()); + CHECK(std::chrono::steady_clock::now() - t0 < std::chrono::seconds(5)); +} + +TEST_CASE("starting a missing program fails with a message") +{ + ChildProcess child; + std::string error; + CHECK_FALSE(child.start("/nonexistent/program", {}, error)); + CHECK_FALSE(error.empty()); +} + +#else // cmd.exe + +namespace +{ + std::filesystem::path cmdExe() + { + const char* comspec = std::getenv("ComSpec"); + return (comspec && *comspec) ? std::filesystem::path(comspec) : std::filesystem::path("C:\\Windows\\System32\\cmd.exe"); + } +} // namespace + +TEST_CASE("stdout and stderr are merged line by line and the exit code is returned") +{ + ChildProcess child; + std::string error; + // Redirection first: in "echo two 1>&2" cmd would print the space before the redirection too. + REQUIRE(child.start(cmdExe(), { "/c", "echo one& 1>&2 echo two& exit /b 3" }, error)); + const auto lines = readAll(child); + CHECK(child.wait() == 3); + REQUIRE(lines.size() == 2); + CHECK(lines[0] == "one"); // '\r' of the CRLF removed + CHECK(lines[1] == "two"); +} + +TEST_CASE("stdin is closed, so a program waiting for input does not hang") +{ + ChildProcess child; + std::string error; + REQUIRE(child.start(cmdExe(), { "/c", "set /p x=& echo got-eof" }, error)); + const auto lines = readAll(child); + REQUIRE(!lines.empty()); + CHECK(lines.back() == "got-eof"); + CHECK(child.wait() != -1); +} + +TEST_CASE("kill stops the child and everything it started promptly") +{ + ChildProcess child; + std::string error; + // ping is a grandchild holding the pipe's write end (stderr): only the job object takes it down too. + REQUIRE(child.start(cmdExe(), { "/c", "echo started& ping -n 30 127.0.0.1 >nul" }, error)); + std::string line; + REQUIRE(child.readLine(line)); + CHECK(line == "started"); + + const auto t0 = std::chrono::steady_clock::now(); + std::thread killer( + [&] + { + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + child.kill(); + }); + while (child.readLine(line)) + { + } + killer.join(); + CHECK(child.wait() == -1); + CHECK(child.wasKilled()); + CHECK(std::chrono::steady_clock::now() - t0 < std::chrono::seconds(5)); +} + +TEST_CASE("starting a missing program fails with a message") +{ + ChildProcess child; + std::string error; + CHECK_FALSE(child.start("C:\\nonexistent\\program.exe", {}, error)); + CHECK_FALSE(error.empty()); +} + +#endif diff --git a/apps/easy_gui/tests/test_export.cpp b/apps/easy_gui/tests/test_export.cpp new file mode 100644 index 00000000..b4e250a5 --- /dev/null +++ b/apps/easy_gui/tests/test_export.cpp @@ -0,0 +1,556 @@ +#include + +#include "export.h" +#include "forestry_preset.h" +#include "las14_writer.h" +#include "test_helpers.h" + +#include + +#include +#include + +#include +#include +#include +#include +#include + +using namespace easy_gui; +namespace fs = std::filesystem; + +namespace +{ + struct ReadPoint + { + Eigen::Vector3d xyz; + double gps_time; + uint16_t intensity; + uint16_t point_source_id; + uint8_t user_data; + float range_m; + uint8_t point_origin; + }; + + struct ReadFile + { + laszip_header header{}; + std::vector extra_names; + std::vector extra_types; + std::vector extra_options; + std::vector points; + }; + + ReadFile readLaz(const fs::path& file) + { + ReadFile r; + laszip_POINTER reader = nullptr; + REQUIRE(laszip_create(&reader) == 0); + laszip_BOOL compressed = 0; + REQUIRE(laszip_open_reader(reader, file.string().c_str(), &compressed) == 0); + laszip_header* h = nullptr; + laszip_get_header_pointer(reader, &h); + r.header = *h; + for (laszip_U32 v = 0; v < h->number_of_variable_length_records; v++) + { + const auto& vlr = h->vlrs[v]; + if (std::string(vlr.user_id) == "LASF_Spec" && vlr.record_id == 4) + for (size_t k = 0; k + 192 <= vlr.record_length_after_header; k += 192) + { + const laszip_U8* d = vlr.data + k; + r.extra_types.push_back(d[2]); + r.extra_options.push_back(d[3]); + r.extra_names.emplace_back(reinterpret_cast(d + 4), strnlen(reinterpret_cast(d + 4), 32)); + } + } + laszip_point* p = nullptr; + laszip_get_point_pointer(reader, &p); + const uint64_t n = h->number_of_point_records ? h->number_of_point_records : h->extended_number_of_point_records; + for (uint64_t i = 0; i < n; i++) + { + REQUIRE(laszip_read_point(reader) == 0); + ReadPoint q{}; + q.xyz = { h->x_offset + h->x_scale_factor * p->X, + h->y_offset + h->y_scale_factor * p->Y, + h->z_offset + h->z_scale_factor * p->Z }; + q.gps_time = p->gps_time; + q.intensity = p->intensity; + q.point_source_id = p->point_source_ID; + q.user_data = p->user_data; + if (p->num_extra_bytes >= 5) + { + std::memcpy(&q.range_m, p->extra_bytes, 4); + q.point_origin = p->extra_bytes[4]; + } + r.points.push_back(q); + } + laszip_close_reader(reader); + laszip_destroy(reader); + return r; + } + + struct RawPoint + { + Eigen::Vector3d xyz; + double t; + uint16_t intensity = 7; + }; + + //! Mandeye-style raw LAZ (LAS 1.2 PF1, lidar frame) with the given points. + void writeRaw(const fs::path& file, const std::vector& points, int lidar_id) + { + laszip_POINTER writer = nullptr; + REQUIRE(laszip_create(&writer) == 0); + laszip_header* header = nullptr; + laszip_get_header_pointer(writer, &header); + header->version_major = 1; + header->version_minor = 2; + header->point_data_format = 1; + header->point_data_record_length = 28; + header->x_scale_factor = header->y_scale_factor = header->z_scale_factor = 0.0001; + REQUIRE(laszip_open_writer(writer, file.string().c_str(), 1) == 0); + laszip_point* point = nullptr; + laszip_get_point_pointer(writer, &point); + for (const auto& p : points) + { + point->gps_time = p.t; + point->user_data = static_cast(lidar_id); + point->intensity = p.intensity; + laszip_F64 xyz[3] = { p.xyz.x(), p.xyz.y(), p.xyz.z() }; + laszip_set_coordinates(writer, xyz); + laszip_write_point(writer); + laszip_update_inventory(writer); + } + laszip_close_writer(writer); + laszip_destroy(writer); + } + + void writePoseFile(const fs::path& file, const std::vector& poses, const std::vector& names) + { + std::ofstream out(file); + out << poses.size() << "\n" << std::setprecision(17); + for (size_t i = 0; i < poses.size(); i++) + { + out << names[i] << "\n"; + const auto& m = poses[i].matrix(); + for (int r = 0; r < 4; r++) + out << m(r, 0) << " " << m(r, 1) << " " << m(r, 2) << " " << m(r, 3) << "\n"; + } + } + + //! A small synthetic step 1 result: kScans scans of kNodes trajectory nodes at 200 Hz along a curved path, written + //! in exactly step 1's file formats (save_result in lidar_odometry.cpp), plus the matching raw recording. + struct SyntheticSession + { + static constexpr int kScans = 2; + static constexpr int kNodes = 40; + static constexpr double kT0 = 1783328634.0; + static constexpr int kLidarId = 3; + + fs::path session_file; + RecordingFiles recording; + std::vector sensor_points; // per scan_lio point, in the sensor frame (for range_m) + size_t odometry_points = 0; + + static double nodeTime(int k) + { + return kT0 + 0.005 * k; + } + //! Global pose of node k (k counts across scans). + static Eigen::Affine3d pose(int k) + { + Eigen::Affine3d m = Eigen::Affine3d::Identity(); + m.linear() = (Eigen::AngleAxisd(0.5 + 0.02 * k, Eigen::Vector3d::UnitZ()) * Eigen::AngleAxisd(0.1, Eigen::Vector3d::UnitX())) + .toRotationMatrix(); + m.translation() = Eigen::Vector3d(10.0 + 0.005 * k, 5.0 + 0.002 * k, 1.0); + return m; + } + + explicit SyntheticSession(const fs::path& root) + { + const fs::path session = root / "lio_session"; + const fs::path raw = root / "recording"; + fs::create_directories(session); + fs::create_directories(raw); + + std::vector scan_poses; + std::vector names; + nlohmann::json laz_names; + for (int s = 0; s < kScans; s++) + { + const int first = s * kNodes; + const Eigen::Affine3d m_pose = pose(first); + scan_poses.push_back(m_pose); + names.push_back("scan_lio_" + std::to_string(s) + ".laz"); + laz_names.push_back({ { "file_name", (session / names.back()).string() } }); + + std::ofstream trj(session / ("trajectory_lio_" + std::to_string(s) + ".csv")); + trj << "timestamp_nanoseconds pose00 pose01 pose02 pose03 pose10 pose11 pose12 pose13 pose20 pose21 pose22 pose23 " + "timestampUnix_nanoseconds om_rad fi_rad ka_rad\n"; + std::vector local_points; + std::vector intensity; + std::vector timestamps; + for (int j = 0; j < kNodes; j++) + { + const Eigen::Affine3d local = m_pose.inverse() * pose(first + j); + trj << std::setprecision(20) << nodeTime(first + j) * 1e9 << " " << std::setprecision(10); + for (int r = 0; r < 3; r++) + for (int c = 0; c < 4; c++) + trj << local(r, c) << " "; + trj << std::setprecision(20) << std::floor(nodeTime(first + j) * 1000) << " 0.01 0.02 0.03 \n"; + + // two odometry points per node, 1 ms after it: one near, one far + for (const Eigen::Vector3d p_sensor : { Eigen::Vector3d(1.2, 0.3, 0.4), Eigen::Vector3d(-8.0, 3.0, 2.0) }) + { + local_points.push_back(local * p_sensor); + intensity.push_back(static_cast(20 + j)); + timestamps.push_back(nodeTime(first + j) + 0.001); // exportLaz writes seconds x 1e9, as step 1 + sensor_points.push_back(p_sensor); + } + } + REQUIRE(exportLaz((session / names.back()).string(), local_points, intensity, timestamps)); + odometry_points += local_points.size(); + } + writePoseFile(session / "session_poses.mrp", scan_poses, names); + writePoseFile(session / "session_ini_poses.mri", scan_poses, names); + std::string error; + REQUIRE(writeStep1ParameterFile(session / "HDMapping_params_test.toml", error)); + + nlohmann::json settings = { { "offset_x", 0.0 }, + { "offset_y", 0.0 }, + { "offset_z", 0.0 }, + { "folder_name", session.string() }, + { "out_folder_name", session.string() }, + { "poses_file_name", (session / "session_poses.mrp").string() }, + { "initial_poses_file_name", (session / "session_ini_poses.mri").string() }, + { "out_poses_file_name", (session / "session_poses.mrp").string() } }; + session_file = session / "session.mjs"; + std::ofstream(session_file) << nlohmann::json{ { "Session Settings", settings }, { "laz_file_names", laz_names } }.dump(2); + + recording.laz_files = { raw / "lidar0000.laz", raw / "lidar0001.laz" }; + recording.csv_files = { raw / "imu0000.csv", raw / "imu0001.csv" }; + recording.sn_file = raw / "lidar0000.sn"; + recording.pairs = 2; + easy_gui_test::writeText(recording.sn_file, std::to_string(kLidarId) + " ARMCP1U0038132\n"); + } + }; + + bool near(const Eigen::Vector3d& a, const Eigen::Vector3d& b, double tolerance = 0.0002) + { + return (a - b).norm() <= tolerance; + } +} // namespace + +TEST_CASE("Las14Writer writes LAS 1.4 point format 6 with named, unscaled extra bytes") +{ + easy_gui_test::TempDir tmp; + const fs::path file = tmp.path / "out.laz"; + Las14Writer writer; + std::string error; + REQUIRE(writer.open( + file.string(), + { { ExtraBytesAttribute::Type::F32, "range_m", "range" }, { ExtraBytesAttribute::Type::U8, "point_origin", "flag" } }, + Eigen::Vector3d(100.0, 200.0, 0.0), + 0.0001, + "test system", + "test", + error)); +#pragma pack(push, 1) + struct + { + float r; + uint8_t f; + } extra{ 12.5f, 1 }; +#pragma pack(pop) + Las14Point p; + p.position = { 101.23456, 198.5, -3.25 }; + p.gps_time = 1783328634.123456; + p.intensity = 77; + p.point_source_id = 4321; + p.user_data = 2; + p.extra_bytes = reinterpret_cast(&extra); + REQUIRE(writer.write(p, error)); + p.extra_bytes = nullptr; // all-zero extra bytes + p.position = { 99.0, 201.0, 5.0 }; + REQUIRE(writer.write(p, error)); + REQUIRE(writer.close(error)); + + const ReadFile r = readLaz(file); + CHECK(r.header.version_major == 1); + CHECK(r.header.version_minor == 4); + CHECK(r.header.point_data_format == 6); + CHECK(r.header.point_data_record_length == 35); + CHECK((r.header.global_encoding & 1) == 0); // gps_time is not adjusted GPS time + CHECK((r.header.global_encoding & 16) == 16); // WKT bit, required for PF6 + CHECK(r.header.extended_number_of_point_records == 2); + CHECK(r.header.number_of_point_records == 0); // legacy counter stays 0 for PF6 + CHECK(r.header.min_x == doctest::Approx(99.0)); + CHECK(r.header.max_z == doctest::Approx(5.0)); + REQUIRE(r.extra_names.size() == 2); + CHECK(r.extra_names[0] == "range_m"); + CHECK(r.extra_names[1] == "point_origin"); + CHECK(r.extra_types[0] == 9); // float + CHECK(r.extra_types[1] == 1); // unsigned char + CHECK(r.extra_options[0] == 0); // no scale: readers return the raw float / uint8 + CHECK(r.extra_options[1] == 0); + REQUIRE(r.points.size() == 2); + CHECK(near(r.points[0].xyz, { 101.2346, 198.5, -3.25 }, 1e-6)); + CHECK(r.points[0].gps_time == 1783328634.123456); + CHECK(r.points[0].intensity == 77); + CHECK(r.points[0].point_source_id == 4321); + CHECK(r.points[0].user_data == 2); + CHECK(r.points[0].range_m == 12.5f); + CHECK(r.points[0].point_origin == 1); + CHECK(r.points[1].range_m == 0.0f); + CHECK(r.points[1].point_origin == 0); +} + +TEST_CASE("exportSession: step 1 points as step 2 places them, near points with step 1's transform chain") +{ + easy_gui_test::TempDir tmp; + SyntheticSession s(tmp.path); + using S = SyntheticSession; + + // Raw points. Node 5 lies in scan 0, node 47 in scan 1 (7th node of that scan). + const double t5 = S::nodeTime(5) + 0.002; + const double t47 = S::nodeTime(47) + 0.004; + const Eigen::Vector3d near_a(0.5, 0.3, 0.2); // horizontal 0.58 m, 3D 0.62 m: near point + const Eigen::Vector3d near_steep(0.6, 0.0, 1.5); // horizontal 0.6 m but 3D 1.62 m: still a near point + const Eigen::Vector3d band(1.1, 0.0, 0.0); // step 1 exports this one itself + // step 1 has 20 nodes per piece (threshold_nr_poses) and skips [last node of a piece, next piece's first node) + const Eigen::Vector3d gap_a(2.5, 1.0, 0.3); + const Eigen::Vector3d gap_b(-4.0, 2.0, 1.0); + const double t_gap_a = S::nodeTime(19) + 0.002; + const double t_gap_b = S::nodeTime(39) + 0.003; + std::vector file0 = { + { near_a, t5, 11 }, + { Eigen::Vector3d(0.2, 0.1, 0.0), t5 }, // 3D 0.22 m < near_min_range + { Eigen::Vector3d::Zero(), t5 }, // no return + { Eigen::Vector3d(0.5, 0.0, 0.0), 0.0 }, // time 0 + { Eigen::Vector3d(3.0, 0.0, 0.0), t5 }, // step 1's point + { band, t5 }, + { near_a, S::kT0 - 1.0 }, // before the trajectory: no pose within 10 ms + { gap_a, t_gap_a, 13 }, // inside step 1's limits, but in the gap after piece 0 (nodes 0-19): gap point + { gap_b, t_gap_b }, // gap at the end of scan 0 (node 39 ends piece 1), before scan 1's first node + { Eigen::Vector3d(2.0, 0.5, 0.1), S::nodeTime(18) + 0.002 }, // inside a piece: step 1's point, not added + }; + std::vector file1 = { + { near_steep, t47, 12 }, + { near_a, S::nodeTime(2 * S::kNodes - 1) + 0.02 }, // 20 ms after the last node + }; + writeRaw(s.recording.laz_files[0], file0, S::kLidarId); + writeRaw(s.recording.laz_files[1], file1, S::kLidarId); + + ExportRequest request; + request.session_file = s.session_file; + request.recording = s.recording; + request.lidar_id = S::kLidarId; + request.near_min_range_m = 0.4; + request.laz_file = tmp.path / "out" / "point_cloud.laz"; + request.trajectory_file = tmp.path / "out" / "trajectory.csv"; + request.readme_file = tmp.path / "out" / "README.txt"; + + std::atomic cancel{ false }; + double last_fraction = -1.0; + bool monotonic = true; + ExportStats stats; + std::string error; + REQUIRE_MESSAGE( + exportSession( + request, + cancel, + [&](double f, const std::string&) + { + monotonic = monotonic && f >= last_fraction; + last_fraction = f; + }, + stats, + error), + error); + CHECK(monotonic); + CHECK(last_fraction == 1.0); + + CHECK(stats.scans == 2); + CHECK(stats.trajectory_nodes == 2 * S::kNodes); + CHECK(stats.odometry_points == s.odometry_points); + CHECK(stats.near_points == 2); + CHECK(stats.gap_points == 2); + CHECK(stats.check_band_points == 0); + CHECK(stats.threshold_nr_poses == 20); + CHECK(stats.raw_points_read == file0.size() + file1.size()); + CHECK(stats.raw_no_return == 2); + CHECK(stats.raw_outside_trajectory == 2); + CHECK(stats.filter_threshold_xy_inner == 1.0); + CHECK(stats.threshould_output_filter == 1.0); // Easy GUI's output setting, read back from the session + + const ReadFile r = readLaz(request.laz_file); + REQUIRE(r.points.size() == s.odometry_points + 4); // raw file order: near_a, gap_a, gap_b, near_steep + + // odometry points: scan pose x local point, range from the sensor frame, ramp time in seconds + for (size_t i = 0; i < s.odometry_points; i++) + { + const auto& p = r.points[i]; + const int scan = static_cast(i / (2 * S::kNodes)); + const int node = scan * S::kNodes + static_cast(i % (2 * S::kNodes)) / 2; + CHECK(p.point_origin == kStep1); + CHECK(p.point_source_id == scan); + CHECK(p.user_data == S::kLidarId); + CHECK(near(p.xyz, S::pose(node) * s.sensor_points[i])); + CHECK(p.range_m == doctest::Approx(s.sensor_points[i].norm()).epsilon(1e-4)); + CHECK(p.gps_time == doctest::Approx(S::nodeTime(node) + 0.001).epsilon(1e-15)); + } + + // near points: pose of the last node before the raw time, raw time and intensity kept + const auto& a = r.points[s.odometry_points]; + CHECK(a.point_origin == kNearPoint); + CHECK(near(a.xyz, S::pose(5) * near_a)); + CHECK(a.gps_time == t5); + CHECK(a.intensity == 11); + CHECK(a.point_source_id == 0); + CHECK(a.range_m == doctest::Approx(near_a.norm())); + // gap points: same placement rule, raw time, scan of the node + const auto& g = r.points[s.odometry_points + 1]; + CHECK(g.point_origin == kGapPoint); + CHECK(near(g.xyz, S::pose(19) * gap_a)); + CHECK(g.gps_time == t_gap_a); + CHECK(g.intensity == 13); + CHECK(g.point_source_id == 0); + CHECK(g.range_m == doctest::Approx(gap_a.norm())); + const auto& g2 = r.points[s.odometry_points + 2]; + CHECK(g2.point_origin == kGapPoint); + CHECK(near(g2.xyz, S::pose(39) * gap_b)); + CHECK(g2.point_source_id == 0); + const auto& b = r.points[s.odometry_points + 3]; + CHECK(b.point_origin == kNearPoint); + CHECK(near(b.xyz, S::pose(47) * near_steep)); + CHECK(b.point_source_id == 1); + CHECK(b.range_m == doctest::Approx(near_steep.norm())); + CHECK(stats.range_min_m == doctest::Approx(near_a.norm()).epsilon(1e-6)); + + // trajectory: one row per node, global pose, Unix seconds + std::ifstream csv(request.trajectory_file); + std::string header, row; + std::getline(csv, header); + CHECK(header.starts_with("timestamp_unix_s,x,y,z,qw,qx,qy,qz")); + int rows = 0; + std::vector row47; + while (std::getline(csv, row)) + { + if (rows == 47) + { + std::stringstream ss(row); + std::string v; + while (std::getline(ss, v, ',')) + row47.push_back(std::stod(v)); + } + rows++; + } + CHECK(rows == 2 * S::kNodes); + REQUIRE(row47.size() == 16); + CHECK(row47[0] == doctest::Approx(S::nodeTime(47)).epsilon(1e-15)); + CHECK(near({ row47[1], row47[2], row47[3] }, S::pose(47).translation())); + const Eigen::Quaterniond q(S::pose(47).rotation()); + CHECK(std::abs(std::abs(q.dot(Eigen::Quaterniond(row47[4], row47[5], row47[6], row47[7]))) - 1.0) < 1e-8); + CHECK(row47[11] == 1); // scan_index + + std::ifstream readme(request.readme_file); + const std::string text((std::istreambuf_iterator(readme)), std::istreambuf_iterator()); + CHECK(text.find("Unix epoch seconds") != std::string::npos); + CHECK(text.find("at least 0.40 m") != std::string::npos); + CHECK(text.find("point_origin") != std::string::npos); + CHECK(text.find("gap points (2)") != std::string::npos); + + SUBCASE("check-band test mode adds the 1.0-1.2 m raw points with flag 2") + { + request.check_band_outer_m = 1.2; + REQUIRE(exportSession( + request, + cancel, + [](double, const std::string&) + { + }, + stats, + error)); + CHECK(stats.near_points == 2); + CHECK(stats.check_band_points == 1); + const ReadFile rb = readLaz(request.laz_file); + CHECK(stats.gap_points == 2); + REQUIRE(rb.points.size() == s.odometry_points + 5); + const auto& c = rb.points[s.odometry_points + 1]; // raw file order: near_a, band, gap_a, gap_b, near_steep + CHECK(c.point_origin == kCheckBand); + CHECK(near(c.xyz, S::pose(5) * band)); + CHECK(c.range_m == doctest::Approx(band.norm())); + } + + SUBCASE("place_by_raw_time takes every point from the raw files, at its raw time") + { + request.place_by_raw_time = true; + request.check_band_outer_m = 1.2; // ignored in this mode + REQUIRE(exportSession( + request, + cancel, + [](double, const std::string&) + { + }, + stats, + error)); + CHECK(stats.odometry_points == 3); // the 3 m, 1.1 m and node-18 points, not scan_lio's points + CHECK(stats.near_points == 2); + CHECK(stats.gap_points == 2); + CHECK(stats.check_band_points == 0); + CHECK(stats.raw_outside_trajectory == 2); + const ReadFile rr = readLaz(request.laz_file); + REQUIRE(rr.points.size() == 7); // raw file order: near_a, 3 m, band, gap_a, gap_b, node 18, near_steep + CHECK(rr.points[0].point_origin == kNearPoint); + CHECK(rr.points[1].point_origin == kStep1); + CHECK(near(rr.points[1].xyz, S::pose(5) * Eigen::Vector3d(3.0, 0.0, 0.0))); + CHECK(rr.points[1].gps_time == t5); + CHECK(rr.points[1].range_m == doctest::Approx(3.0)); + CHECK(rr.points[2].point_origin == kStep1); + CHECK(near(rr.points[2].xyz, S::pose(5) * band)); + CHECK(rr.points[3].point_origin == kGapPoint); + CHECK(rr.points[5].point_origin == kStep1); + CHECK(near(rr.points[5].xyz, S::pose(18) * Eigen::Vector3d(2.0, 0.5, 0.1))); + CHECK(rr.points[6].point_origin == kNearPoint); + CHECK(rr.points[6].point_source_id == 1); + } + + SUBCASE("cancel stops the export") + { + cancel = true; + CHECK_FALSE(exportSession( + request, + cancel, + [](double, const std::string&) + { + }, + stats, + error)); + CHECK(error.find("cancelled") != std::string::npos); + } +} + +TEST_CASE("exportSession refuses a session without its parameter file, in plain language") +{ + easy_gui_test::TempDir tmp; + SyntheticSession s(tmp.path); + fs::remove(s.session_file.parent_path() / "HDMapping_params_test.toml"); + ExportRequest request; + request.session_file = s.session_file; + request.recording = s.recording; + request.laz_file = tmp.path / "point_cloud.laz"; + request.trajectory_file = tmp.path / "trajectory.csv"; + request.readme_file = tmp.path / "README.txt"; + std::atomic cancel{ false }; + ExportStats stats; + std::string error; + CHECK_FALSE(exportSession( + request, + cancel, + [](double, const std::string&) + { + }, + stats, + error)); + CHECK(error.find("settings file") != std::string::npos); +} diff --git a/apps/easy_gui/tests/test_helpers.h b/apps/easy_gui/tests/test_helpers.h new file mode 100644 index 00000000..ec73fdcd --- /dev/null +++ b/apps/easy_gui/tests/test_helpers.h @@ -0,0 +1,97 @@ +#pragma once + +#include + +#include +#include +#include +#include +#include +#include + +namespace easy_gui_test +{ + //! Fresh empty directory under the system temp dir, removed again in the destructor. + struct TempDir + { + std::filesystem::path path; + TempDir() + { + static std::atomic counter{ 0 }; + path = std::filesystem::temp_directory_path() / + ("easy_gui_test_" + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count()) + "_" + + std::to_string(counter++)); + std::filesystem::create_directories(path); + } + ~TempDir() + { + std::error_code ec; + std::filesystem::remove_all(path, ec); + } + }; + + //! Mandeye-style raw LAZ (LAS 1.2 PF1): `n` points starting at gps_time `t0` [s], 1 ms apart. + inline bool writeRawLaz(const std::filesystem::path& file, int n, double t0, int lidar_id = 0) + { + laszip_POINTER writer = nullptr; + if (laszip_create(&writer)) + return false; + laszip_header* header = nullptr; + laszip_get_header_pointer(writer, &header); + header->version_major = 1; + header->version_minor = 2; + header->point_data_format = 1; + header->point_data_record_length = 28; + header->x_scale_factor = header->y_scale_factor = header->z_scale_factor = 0.0001; + if (laszip_open_writer(writer, file.string().c_str(), 1)) + return false; + laszip_point* point = nullptr; + laszip_get_point_pointer(writer, &point); + for (int i = 0; i < n; i++) + { + point->gps_time = t0 + i * 0.001; + point->user_data = static_cast(lidar_id); + point->intensity = 10; + laszip_F64 xyz[3] = { 2.0 + i * 0.01, 1.0, 0.5 }; + laszip_set_coordinates(writer, xyz); + laszip_write_point(writer); + laszip_update_inventory(writer); + } + laszip_close_writer(writer); + laszip_destroy(writer); + return true; + } + + //! Mandeye IMU csv with named columns, `n` samples at 200 Hz starting at `t0` [s] (written in ns). + inline void writeImuCsv(const std::filesystem::path& file, int n, double t0) + { + std::ofstream out(file); + out << "timestamp gyroX gyroY gyroZ accX accY accZ imuId timestampUnix\n"; + for (int i = 0; i < n; i++) + { + const auto ns = static_cast((t0 + i * 0.005) * 1e9); + out << ns << " 0 0 0 0 0 1 0 " << ns / 1000000 << "\n"; + } + } + + inline void writeText(const std::filesystem::path& file, const std::string& text) + { + std::ofstream out(file, std::ios::binary); + out << text; + } + + //! A plausible two-pair recording (plus an unmatched trailing lidar file, like real recordings often have), + //! 60 s long so it does not trigger the short-recording warning. + inline void writeRecording(const std::filesystem::path& dir, double t0 = 1783328629.0) + { + std::filesystem::create_directories(dir); + writeRawLaz(dir / "lidar0000.laz", 2000, t0); + writeImuCsv(dir / "imu0000.csv", 6000, t0); + writeRawLaz(dir / "lidar0001.laz", 2000, t0 + 30.0); + writeImuCsv(dir / "imu0001.csv", 6000, t0 + 30.0); + writeRawLaz(dir / "lidar0002.laz", 100, t0 + 60.0); + writeText(dir / "lidar0000.sn", "0 ARMCP1U0038132\n"); + writeText(dir / "lidar0001.sn", "0 ARMCP1U0038132\n"); + writeText(dir / "status0000.json", "{}"); + } +} // namespace easy_gui_test diff --git a/apps/easy_gui/tests/test_main.cpp b/apps/easy_gui/tests/test_main.cpp new file mode 100644 index 00000000..7207bc37 --- /dev/null +++ b/apps/easy_gui/tests/test_main.cpp @@ -0,0 +1,20 @@ +#define DOCTEST_CONFIG_IMPLEMENT +#include + +#include +#include + +// `easy_gui_tests --echo-args ` prints each argument as "[arg]" on its own line and exits: a child program +// that behaves the same on every platform, for the ChildProcess argument quoting test. +int main(int argc, char** argv) +{ + if (argc > 1 && std::strcmp(argv[1], "--echo-args") == 0) + { + for (int i = 2; i < argc; i++) + std::printf("[%s]\n", argv[i]); + return 0; + } + doctest::Context context; + context.applyCommandLine(argc, argv); + return context.run(); +} diff --git a/apps/easy_gui/tests/test_pipeline_integration.cpp b/apps/easy_gui/tests/test_pipeline_integration.cpp new file mode 100644 index 00000000..d9275f49 --- /dev/null +++ b/apps/easy_gui/tests/test_pipeline_integration.cpp @@ -0,0 +1,215 @@ +#include + +#include "batch.h" +#include "pipeline.h" +#include "test_helpers.h" + +#include +#include +#include + +using namespace easy_gui; +namespace fs = std::filesystem; + +// Integration tests against the real lidar_odometry_step_1 (built next to this test binary). They need a real +// Mandeye recording, so they only run when EASY_GUI_TEST_RECORDING points to one (a few file pairs are enough). +namespace +{ + const char* testRecording() + { + return std::getenv("EASY_GUI_TEST_RECORDING"); + } + + PipelineStatus waitUntil(const Pipeline& pipeline, bool (*done)(const PipelineStatus&), std::chrono::seconds timeout) + { + const auto t0 = std::chrono::steady_clock::now(); + PipelineStatus s = pipeline.status(); + while (!done(s) && std::chrono::steady_clock::now() - t0 < timeout) + { + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + s = pipeline.status(); + } + return s; + } +} // namespace + +TEST_CASE("cancel stops a real step 1 run during odometry and cleans up" * doctest::skip(testRecording() == nullptr)) +{ + easy_gui_test::TempDir tmp; + const fs::path out = tmp.path / "out"; + Pipeline pipeline; + std::string error; + REQUIRE(pipeline.start(PipelineRequest{ testRecording(), out, findStep1Executable(), Settings{} }, error)); + + const PipelineStatus during = waitUntil( + pipeline, + [](const PipelineStatus& s) + { + return s.state != PipelineState::Running || (s.step1.phase == Step1Phase::Odometry && s.step1.chunks_done > 0); + }, + std::chrono::seconds(300)); + REQUIRE(during.state == PipelineState::Running); + REQUIRE(during.step1.phase == Step1Phase::Odometry); + + const auto t_cancel = std::chrono::steady_clock::now(); + pipeline.cancel(); + const PipelineStatus after = waitUntil( + pipeline, + [](const PipelineStatus& s) + { + return s.state != PipelineState::Running; + }, + std::chrono::seconds(30)); + CHECK(after.state == PipelineState::Cancelled); + CHECK(std::chrono::steady_clock::now() - t_cancel < std::chrono::seconds(10)); + CHECK_FALSE(fs::exists(out / "work")); // links, step 1 cache and partial session are gone + CHECK(fs::exists(out / "easy_gui.log")); // the log stays + CHECK(fs::exists(fs::path(testRecording()))); // and the recording, of course +} + +namespace +{ + //! One file pair of real data, cut to its first 200 points: it passes the pre-flight check, so step 1 really runs, + //! finds no usable data and must fail gracefully. + void writeTooShortRecording(const fs::path& rec) + { + fs::create_directories(rec); + const fs::path source(testRecording()); + fs::copy_file(source / "imu0000.csv", rec / "imu0000.csv"); + fs::copy_file(source / "lidar0000.sn", rec / "lidar0000.sn"); + // Timestamps inside the IMU range, so the pre-flight check passes and step 1 really runs on unusable data. + std::ifstream imu(rec / "imu0000.csv"); + std::string header, first_sample; + std::getline(imu, header); + std::getline(imu, first_sample); + const double imu_t0 = std::stod(first_sample.substr(0, first_sample.find(' '))) / 1e9; + easy_gui_test::writeRawLaz(rec / "lidar0000.laz", 200, imu_t0 + 1.0); + } +} // namespace + +TEST_CASE("a recording too short for odometry fails with a message, not a crash" * doctest::skip(testRecording() == nullptr)) +{ + easy_gui_test::TempDir tmp; + const fs::path rec = tmp.path / "rec"; + writeTooShortRecording(rec); + + Pipeline pipeline; + std::string error; + REQUIRE(pipeline.start(PipelineRequest{ rec, tmp.path / "out", findStep1Executable(), Settings{} }, error)); + const PipelineStatus s = waitUntil( + pipeline, + [](const PipelineStatus& st) + { + return st.state != PipelineState::Running; + }, + std::chrono::seconds(120)); + CHECK(s.state == PipelineState::Failed); + CHECK_FALSE(s.message.empty()); + MESSAGE("message shown to the user: " << s.message); +} + +TEST_CASE("a real run ends with the point cloud, the trajectory and the README" * doctest::skip(testRecording() == nullptr)) +{ + easy_gui_test::TempDir tmp; + const fs::path out = tmp.path / "out"; + Pipeline pipeline; + std::string error; + REQUIRE(pipeline.start(PipelineRequest{ testRecording(), out, findStep1Executable(), Settings{} }, error)); + const PipelineStatus s = waitUntil( + pipeline, + [](const PipelineStatus& st) + { + return st.state != PipelineState::Running; + }, + std::chrono::seconds(600)); + REQUIRE_MESSAGE(s.state == PipelineState::Succeeded, s.message); + const OutputLayout layout(out); + CHECK(fs::file_size(layout.point_cloud) == s.export_stats.laz_bytes); + CHECK(fs::file_size(layout.trajectory) == s.export_stats.trajectory_bytes); + CHECK(fs::exists(layout.readme)); + CHECK(s.export_stats.odometry_points > 0); + CHECK(s.export_stats.near_points > 0); + CHECK(s.export_stats.gap_points > 0); + CHECK(s.export_stats.check_band_points == 0); + CHECK(s.export_stats.range_min_m >= Settings{}.near_min_range_m); + CHECK(fs::exists(layout.session / "session.mjs")); // kept for step 2 + CHECK_FALSE(fs::exists(layout.staging)); +} + +TEST_CASE( + "batch: two copies of a real recording and a broken one; the broken one fails, a restart skips the rest" * + doctest::skip(testRecording() == nullptr)) +{ + easy_gui_test::TempDir tmp; + const fs::path parent = tmp.path / "field_day"; + for (const char* copy : { "a_copy", "c_copy" }) + { + fs::create_directories(parent / copy); + for (const auto& entry : fs::directory_iterator(testRecording())) + if (entry.is_regular_file()) + fs::create_symlink(fs::canonical(entry.path()), parent / copy / entry.path().filename()); + } + writeTooShortRecording(parent / "b_broken"); + fs::create_directories(parent / "notes"); + const fs::path out = tmp.path / "results"; + + BatchPlan plan = planBatch(parent, out); + REQUIRE(plan.request.items.size() == 3); + REQUIRE(plan.request.skipped.size() == 1); + for (const auto& item : plan.request.items) + REQUIRE_MESSAGE(item.include, item.exclude_reason); + plan.request.step1_executable = findStep1Executable(); + + { + BatchRunner runner; + std::string error; + REQUIRE(runner.start(plan.request, error)); + runner.wait(); + const BatchStatus s = runner.status(); + CHECK_MESSAGE(s.items[0].state == BatchItemState::Succeeded, s.items[0].error); + CHECK(s.items[1].state == BatchItemState::Failed); + CHECK_FALSE(s.items[1].error.empty()); + CHECK_MESSAGE(s.items[2].state == BatchItemState::Succeeded, s.items[2].error); + MESSAGE("broken recording: " << s.items[1].error); + for (const char* copy : { "a_copy", "c_copy" }) + { + const OutputLayout layout(out / copy); + CHECK(fs::file_size(layout.point_cloud) > 0); + CHECK(fs::exists(layout.trajectory)); + CHECK(fs::exists(layout.done_marker)); + CHECK_FALSE(fs::exists(layout.staging)); + } + CHECK(s.items[0].exported_points == s.items[2].exported_points); // step 1 is deterministic + CHECK(fs::exists(out / "b_broken" / "easy_gui.log")); + CHECK_FALSE(fs::exists(OutputLayout(out / "b_broken").done_marker)); + CHECK(fs::exists(out / BatchRunner::kLogFileName)); + } + + // Restart: the copies are skipped, the broken one is retried (and fails again). + BatchPlan again = planBatch(parent, out); + REQUIRE(again.request.items.size() == 3); + CHECK(again.checks[0].done.has_value()); + CHECK_FALSE(again.checks[1].done.has_value()); + CHECK(again.request.items[1].include); // its unfinished attempt (the log) does not block it + again.request.step1_executable = findStep1Executable(); + BatchRunner runner; + std::string error; + REQUIRE(runner.start(again.request, error)); + runner.wait(); + const BatchStatus s = runner.status(); + CHECK(s.items[0].state == BatchItemState::AlreadyDone); + CHECK(s.items[1].state == BatchItemState::Failed); + CHECK(s.items[2].state == BatchItemState::AlreadyDone); + + std::ifstream csv(out / BatchRunner::kSummaryFileName); + std::string header, a, b, c, notes; + std::getline(csv, header); + std::getline(csv, a); + std::getline(csv, b); + std::getline(csv, c); + std::getline(csv, notes); + CHECK(a.find(",a_copy,already_done,") != std::string::npos); + CHECK(b.find(",b_broken,failed,") != std::string::npos); + CHECK(c.find(",c_copy,already_done,") != std::string::npos); + CHECK(notes.find(",,skipped,") != std::string::npos); +} diff --git a/apps/easy_gui/tests/test_preflight.cpp b/apps/easy_gui/tests/test_preflight.cpp new file mode 100644 index 00000000..0e782506 --- /dev/null +++ b/apps/easy_gui/tests/test_preflight.cpp @@ -0,0 +1,185 @@ +#include + +#include "pipeline.h" +#include "preflight.h" +#include "test_helpers.h" + +#include + +using namespace easy_gui; +using easy_gui_test::TempDir; +namespace fs = std::filesystem; + +namespace +{ + bool hasFinding(const PreflightResult& r, Severity s, const std::string& text) + { + return std::any_of( + r.findings.begin(), + r.findings.end(), + [&](const Finding& f) + { + return f.severity == s && f.message.find(text) != std::string::npos; + }); + } +} // namespace + +TEST_CASE("classifyRecordingFiles pairs like load_data") +{ + TempDir tmp; + easy_gui_test::writeRecording(tmp.path / "rec"); + const auto f = classifyRecordingFiles(tmp.path / "rec"); + CHECK(f.laz_files.size() == 3); + CHECK(f.csv_files.size() == 2); + CHECK(f.pairs == 2); + CHECK(f.sn_file.filename() == "lidar0000.sn"); + CHECK(f.calibration_file.empty()); // status*.json is not a calibration +} + +TEST_CASE("a plausible recording passes with a warning about the unmatched file") +{ + TempDir tmp; + easy_gui_test::writeRecording(tmp.path / "rec"); + const auto r = runPreflight(tmp.path / "rec", tmp.path / "out"); + CHECK(r.ok()); + CHECK(r.lidar_id == 0); + CHECK(r.total_points == 4000); + CHECK(r.duration_s == doctest::Approx(60.0).epsilon(0.01)); + CHECK(hasFinding(r, Severity::Ok, "Recording looks good")); + CHECK(hasFinding(r, Severity::Warning, "lidar0002.laz")); + + // Disk estimate: 120 B per raw point of work space + 2.25x the paired raw LAZ size of results, always reported. + const uint64_t laz_bytes = fs::file_size(tmp.path / "rec" / "lidar0000.laz") + fs::file_size(tmp.path / "rec" / "lidar0001.laz"); + CHECK(r.raw_laz_bytes == laz_bytes); + CHECK(r.estimated_work_bytes == 4000 * 120); + CHECK(r.estimated_result_bytes == laz_bytes * 9 / 4); + CHECK(r.estimated_disk_bytes == r.estimated_work_bytes + r.estimated_result_bytes); + CHECK(hasFinding(r, Severity::Ok, "Disk space: about")); +} + +TEST_CASE("errors are reported in plain language") +{ + TempDir tmp; + const fs::path rec = tmp.path / "rec"; + + SUBCASE("missing folder") + { + const auto r = runPreflight(tmp.path / "nope", tmp.path / "out"); + CHECK_FALSE(r.ok()); + CHECK(hasFinding(r, Severity::Error, "does not exist")); + } + SUBCASE("not a recording") + { + fs::create_directories(rec); + easy_gui_test::writeText(rec / "notes.txt", "hello"); + const auto r = runPreflight(rec, tmp.path / "out"); + CHECK_FALSE(r.ok()); + CHECK(hasFinding(r, Severity::Error, "does not look like a Mandeye recording")); + } + SUBCASE("damaged point cloud file") + { + easy_gui_test::writeRecording(rec); + easy_gui_test::writeText(rec / "lidar0001.laz", "this is not a laz file"); + const auto r = runPreflight(rec, tmp.path / "out"); + CHECK_FALSE(r.ok()); + CHECK(hasFinding(r, Severity::Error, "lidar0001.laz")); + } + SUBCASE("two lidars") + { + easy_gui_test::writeRecording(rec); + easy_gui_test::writeRawLaz(rec / "lidar0001.laz", 2000, 1783328659.0, 1); + const auto r = runPreflight(rec, tmp.path / "out"); + CHECK_FALSE(r.ok()); + CHECK(hasFinding(r, Severity::Error, "more than one lidar")); + } + SUBCASE("clocks do not overlap") + { + easy_gui_test::writeRecording(rec); + easy_gui_test::writeImuCsv(rec / "imu0000.csv", 6000, 1000.0); + easy_gui_test::writeImuCsv(rec / "imu0001.csv", 6000, 1030.0); + const auto r = runPreflight(rec, tmp.path / "out"); + CHECK_FALSE(r.ok()); + CHECK(hasFinding(r, Severity::Error, "do not overlap")); + } + SUBCASE("output inside the recording") + { + easy_gui_test::writeRecording(rec); + const auto r = runPreflight(rec, rec / "results"); + CHECK_FALSE(r.ok()); + CHECK(hasFinding(r, Severity::Error, "inside the recording folder")); + } + SUBCASE("output folder not empty") + { + easy_gui_test::writeRecording(rec); + fs::create_directories(tmp.path / "out"); + easy_gui_test::writeText(tmp.path / "out" / "old.txt", "x"); + const auto r = runPreflight(rec, tmp.path / "out"); + CHECK_FALSE(r.ok()); + CHECK(hasFinding(r, Severity::Error, "already contains files")); + } + SUBCASE("calibration for another scanner") + { + easy_gui_test::writeRecording(rec); + easy_gui_test::writeText( + rec / "calibration.json", R"({"calibration": {"OTHERSERIAL": {"identity": "true"}}, "imuToUse": "OTHERSERIAL"})"); + const auto r = runPreflight(rec, tmp.path / "out"); + CHECK_FALSE(r.ok()); + CHECK(hasFinding(r, Severity::Error, "different scanner")); + } +} + +TEST_CASE("isSameOrInside") +{ + CHECK(isSameOrInside("/a/b/c", "/a/b")); + CHECK(isSameOrInside("/a/b", "/a/b")); + CHECK(isSameOrInside("/a/b/", "/a/b")); + CHECK_FALSE(isSameOrInside("/a/bc", "/a/b")); + CHECK_FALSE(isSameOrInside("/a", "/a/b")); +} + +TEST_CASE("resume_unfinished accepts an output folder that holds only an earlier attempt's own files") +{ + TempDir tmp; + easy_gui_test::writeRecording(tmp.path / "rec"); + const fs::path out = tmp.path / "out"; + fs::create_directories(out / "work" / "lio_session"); + easy_gui_test::writeText(out / "easy_gui.log", "earlier attempt\n"); + easy_gui_test::writeText(out / "point_cloud.laz", "half written"); + CHECK(holdsUnfinishedOutput(out)); + + CHECK_FALSE(runPreflight(tmp.path / "rec", out).ok()); // single mode: still refused + const auto r = runPreflight(tmp.path / "rec", out, true); + CHECK(r.ok()); + CHECK(hasFinding(r, Severity::Ok, "unfinished earlier attempt")); + + SUBCASE("a finished batch result (done marker) is not unfinished") + { + easy_gui_test::writeText(out / "easy_gui_done.txt", "done"); + CHECK_FALSE(holdsUnfinishedOutput(out)); + CHECK_FALSE(runPreflight(tmp.path / "rec", out, true).ok()); + } + SUBCASE("anything else in the folder is not ours") + { + easy_gui_test::writeText(out / "notes.txt", "mine"); + CHECK_FALSE(holdsUnfinishedOutput(out)); + CHECK_FALSE(runPreflight(tmp.path / "rec", out, true).ok()); + } + SUBCASE("missing and empty folders are not unfinished attempts") + { + CHECK_FALSE(holdsUnfinishedOutput(tmp.path / "missing")); + fs::create_directories(tmp.path / "empty"); + CHECK_FALSE(holdsUnfinishedOutput(tmp.path / "empty")); + } +} + +TEST_CASE("holdsUnfinishedOutput knows every file of OutputLayout except the done marker") +{ + TempDir tmp; + const OutputLayout out(tmp.path / "out"); + fs::create_directories(out.session); + for (const auto& file : { out.log_file, out.point_cloud, out.trajectory, out.readme, out.step1_parameters }) + easy_gui_test::writeText(file, "x"); + CHECK(holdsUnfinishedOutput(out.root)); + easy_gui_test::writeText(out.done_marker, "x"); + CHECK_FALSE(holdsUnfinishedOutput(out.root)); +} diff --git a/apps/easy_gui/tests/test_progress_parser.cpp b/apps/easy_gui/tests/test_progress_parser.cpp new file mode 100644 index 00000000..cea44a56 --- /dev/null +++ b/apps/easy_gui/tests/test_progress_parser.cpp @@ -0,0 +1,157 @@ +#include + +#include "progress_parser.h" + +#include +#include + +using namespace easy_gui; + +namespace +{ + // Excerpt of a real lidar_odometry_step_1 CLI log (forest walk, 115 file pairs), paths shortened. The order and + // the interleaving of the per-file lines (parallel loading) are as captured. + const std::vector kRealLogStart = { + "SUCCESS: Version compatible: 0.105.0", + "Parameters loaded OK from: /out/work/easy_gui_step1.toml", + "\"/out/work/staging/lidar0080.laz\"", + "Loading data..", + "--------------", + "Only using first SN file found", + "There is no calibration file in folder!", + "IGNORE THIS MESSAGE IF YOU ONLY HAVE 1 LiDAR", + "loading IMU data from 'imu****.csv' using ID 0 from reference '/out/work/staging/lidar0000.sn' ..", + "Loading point clouds (115 files)..", + "total - filter = kept", + "-----------------------", + "header->number_of_point_records: 1008288", + "counter_filtered_points: 141284", + "1008288 - 141284 = 867004", + "header->number_of_point_records: 1002336", + "counter_filtered_points: 184105", + "1002336 - 184105 = 818231", + "counter_filtered_points: 189379", // its header line was interleaved away + "1000704 - 189379 = 811325", + }; + + const std::vector kRealLogMiddle = { + "TOTAL: 93371048 (lazy: ranges recorded, clouds loaded on demand)", + "..loading finished.", + "start calculating trajectory..", + "AHRS: Fusion (gain=0.5, samples=115959)", + "number of poses: 114944", + "Creating folder: \"/out/work/staging/cache\"", + "running iterations: 1/5747", + "running iterations: 2851/5747", + "[2026-10-17 18:40:42.920] [info] finished optimizing (since start : 0.100 [s]) - 1 / 5747 at iteration 172 / 500 in (load " + "vector 0.012 [s], step1 0.000 [s], step2 0.000 [s], odometry 0.066 [s]) with acc_distance 0.000[m], delta<1.0e-12 " + "(converged)", + "[2026-10-17 18:47:02.538] [info] finished optimizing (since start : 379.716 [s]) - 1810 / 5747 at iteration 182 / 500 in " + "(load vector 0.003 [s], step1 0.000 [s], step2 0.000 [s], odometry 0.195 [s]) with acc_distance 157.907[m], delta<1.0e-12 " + "(converged)", + }; +} // namespace + +TEST_CASE("phases and numbers from a real log") +{ + ProgressParser parser; + CHECK(parser.progress().phase == Step1Phase::Starting); + CHECK(parser.progress().phaseFraction() < 0.0); + + for (const auto& line : kRealLogStart) + parser.feed(line); + CHECK(parser.progress().phase == Step1Phase::LoadingFiles); + CHECK(parser.progress().files_total == 115); + CHECK(parser.progress().files_done == 3); + CHECK(parser.progress().phaseFraction() == doctest::Approx(3.0 / 115.0)); + + for (size_t i = 0; i < 4; i++) + parser.feed(kRealLogMiddle[i]); + CHECK(parser.progress().points_kept == 93371048); + CHECK(parser.progress().phase == Step1Phase::ImuTrajectory); + CHECK(parser.progress().phaseFraction() < 0.0); // no numbers: the GUI shows an indeterminate bar + + for (size_t i = 4; i < 8; i++) + parser.feed(kRealLogMiddle[i]); + CHECK(parser.progress().phase == Step1Phase::Preparing); + CHECK(parser.progress().prepare_done == 2851); + CHECK(parser.progress().prepare_total == 5747); + + parser.feed(kRealLogMiddle[8]); + parser.feed(kRealLogMiddle[9]); + CHECK(parser.progress().phase == Step1Phase::Odometry); + CHECK(parser.progress().chunks_done == 1810); + CHECK(parser.progress().chunks_total == 5747); + const double overall = parser.progress().overallFraction(); + CHECK(overall > 0.10); + CHECK(overall < 0.50); + CHECK_FALSE(parser.progress().failed); +} + +TEST_CASE("saving and finishing") +{ + ProgressParser parser; + parser.feed("[2026-10-17 18:40:42.920] [info] finished optimizing (since start : 1.0 [s]) - 102 / 102 at iteration 1 / 500 in x"); + parser.feed("original_points_to_save_cache_file_name \"/out/work/staging/cache/session_chunk_0.cache\""); + CHECK(parser.progress().phase == Step1Phase::Saving); + parser.feed("saving to: \"/out/work/lio_session/scan_lio_0.laz\""); + CHECK(parser.progress().scans_saved == 1); + CHECK(parser.progress().phaseFraction() == doctest::Approx(0.5)); // 102 chunks -> 2 scan files + parser.feed("remove cache: '/out/work/staging/cache' FINISHED"); + CHECK(parser.progress().phase == Step1Phase::Finished); + CHECK(parser.progress().overallFraction() == doctest::Approx(1.0)); +} + +TEST_CASE("phases never go backwards") +{ + ProgressParser parser; + parser.feed("[x] [info] finished optimizing (since start : 1.0 [s]) - 5 / 10 at iteration 1 / 500 in x"); + parser.feed("running iterations: 1/10"); // a late line from an earlier phase + CHECK(parser.progress().phase == Step1Phase::Odometry); +} + +TEST_CASE("known failures become plain-language messages") +{ + struct Case + { + const char* line; + const char* expected_fragment; + }; + const Case cases[] = { + { "MISSING CALIBRATION FOR imuSnToUse: ABC!!!", "calibration file does not match" }, + { "WRONG CALIBRATION FILE! THE SERIAL NUMBER SHOULD BE X!!!", "different scanner" }, + { "System is out of memory : std::bad_alloc", "ran out of memory" }, + { "Loading poses went wrong! Could not load poses!", "IMU" }, + { "[2026-10-17] [error] points_global spill: write failed", "disk is probably full" }, + { "lazy_load_raw_clouds: '/x/lidar0001.laz' changed or vanished since it was first read", "USB stick" }, + { "Lidar odometry failed: no results saved", "did not succeed" }, + }; + for (const auto& c : cases) + { + CAPTURE(c.line); + ProgressParser parser; + parser.feed(c.line); + CHECK(parser.progress().failed); + CHECK(parser.progress().failure.find(c.expected_fragment) != std::string::npos); + CHECK(parser.progress().last_error_line == c.line); + } + + SUBCASE("the first cause wins") + { + ProgressParser parser; + parser.feed("MISSING CALIBRATION FOR imuSnToUse: ABC!!!"); + parser.feed("Lidar odometry failed: no results saved"); + CHECK(parser.progress().failure.find("calibration") != std::string::npos); + } +} + +TEST_CASE("unknown and malformed lines are ignored") +{ + ProgressParser parser; + parser.feed(""); + parser.feed("running iterations: x/y"); + parser.feed("finished optimizing - a / b at iteration"); + parser.feed("Loading point clouds (lots files).."); + CHECK(parser.progress().phase == Step1Phase::Starting); + CHECK_FALSE(parser.progress().failed); +} diff --git a/apps/easy_gui/tests/test_staging_and_preset.cpp b/apps/easy_gui/tests/test_staging_and_preset.cpp new file mode 100644 index 00000000..eb46ab35 --- /dev/null +++ b/apps/easy_gui/tests/test_staging_and_preset.cpp @@ -0,0 +1,66 @@ +#include + +#include "forestry_preset.h" +#include "staging.h" +#include "test_helpers.h" + +#include + +#include + +using namespace easy_gui; +using easy_gui_test::TempDir; +namespace fs = std::filesystem; + +TEST_CASE("staging links exactly the files step 1 reads and never touches the recording") +{ + TempDir tmp; + const fs::path rec = tmp.path / "rec"; + easy_gui_test::writeRecording(rec); + const auto files = classifyRecordingFiles(rec); + + std::string method, error; + REQUIRE(createStagingFolder(files, tmp.path / "staging", method, error)); + CHECK(fs::exists(tmp.path / "staging" / "lidar0000.laz")); + CHECK(fs::exists(tmp.path / "staging" / "imu0001.csv")); + CHECK(fs::exists(tmp.path / "staging" / "lidar0000.sn")); + CHECK_FALSE(fs::exists(tmp.path / "staging" / "lidar0002.laz")); // unpaired: load_data would skip it anyway + CHECK_FALSE(fs::exists(tmp.path / "staging" / "status0000.json")); + + // The staged view reads the same data. + CHECK(fs::file_size(tmp.path / "staging" / "lidar0001.laz") == fs::file_size(rec / "lidar0001.laz")); + + // Removing the staging folder (as the pipeline does) leaves the recording intact. + fs::remove_all(tmp.path / "staging"); + CHECK(fs::exists(rec / "lidar0000.laz")); + CHECK(fs::exists(rec / "imu0000.csv")); + + SUBCASE("an existing staging folder is refused") + { + fs::create_directories(tmp.path / "staging2"); + CHECK_FALSE(createStagingFolder(files, tmp.path / "staging2", method, error)); + } +} + +TEST_CASE("the step 1 parameter file is preset 2 plus the output-only filter change") +{ + TempDir tmp; + std::string error; + REQUIRE(writeStep1ParameterFile(tmp.path / "p.toml", error)); + + auto params = std::make_unique(); + TomlIO toml_io; + REQUIRE(toml_io.LoadParametersFromTomlFile((tmp.path / "p.toml").string(), *params)); + + CHECK(params->decimation == doctest::Approx(0.01)); + CHECK(params->in_out_params_indoor.resolution_X == doctest::Approx(0.1)); + CHECK(params->in_out_params_outdoor.resolution_Z == doctest::Approx(0.3)); + CHECK(params->filter_threshold_xy_inner == doctest::Approx(1.0)); + CHECK(params->filter_threshold_xy_outer == doctest::Approx(70.0)); + CHECK(params->nr_iter == 500); + CHECK(params->sliding_window_trajectory_length_threshold == doctest::Approx(10000.0)); + CHECK(params->max_distance_lidar == doctest::Approx(70.0)); + CHECK_FALSE(params->use_robust_and_accurate_lidar_odometry); + // The preset says 1.5; Easy GUI lowers it to the load filter (output-only, see forestry_preset.h). + CHECK(params->threshould_output_filter == doctest::Approx(params->filter_threshold_xy_inner)); +} diff --git a/cmake/dependencies.cmake b/cmake/dependencies.cmake index 7a81177c..a8379055 100644 --- a/cmake/dependencies.cmake +++ b/cmake/dependencies.cmake @@ -147,9 +147,13 @@ message(STATUS "Using bundled Xerces-C + libE57Format (static) for E57 support") # ============================================================================ # OpenCV - Computer vision library -include(${THIRDPARTY_DIRECTORY_BINARY}/OpenCV/CMakeLists.txt) -find_package(OpenCV REQUIRED) -message(STATUS "OpenCV include dir: ${OpenCV_INCLUDE_DIRS}, OpenCV libs: ${OpenCV_LIBS}") +# Only the camera/raw-viewer apps link it, and HDMAPPING_EASY_GUI_ONLY never configures those, so that build skips +# it: no Homebrew OpenCV on macOS (built for the runner's macOS version), no ~200 MB download on Windows. +if(NOT HDMAPPING_EASY_GUI_ONLY) + include(${THIRDPARTY_DIRECTORY_BINARY}/OpenCV/CMakeLists.txt) + find_package(OpenCV REQUIRED) + message(STATUS "OpenCV include dir: ${OpenCV_INCLUDE_DIRS}, OpenCV libs: ${OpenCV_LIBS}") +endif() # PROJ - Cartographic projections library include(${THIRDPARTY_DIRECTORY_BINARY}/Proj/CMakeLists.txt) diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt index 5e38ab80..d6a90b0f 100644 --- a/core/CMakeLists.txt +++ b/core/CMakeLists.txt @@ -116,17 +116,17 @@ function(add_core_target target_name with_gui) endif() endfunction() -# core_no_gui is used by python bindings, lidar odometry test subprojects and session_to_mcap (console tools) only - if they are not build just do not waste time building core_no_gui -if(BUILD_TESTING OR BUILD_WITH_PYBIND OR BUILD_WITH_CLI_TOOLS) - add_core_target(core_no_gui FALSE) +# core_no_gui is used by python bindings, lidar odometry test subprojects, session_to_mcap (console tools) and +# apps/easy_gui. Easy GUI is always built (its export reads sessions with Session), so core_no_gui is too; it was +# previously only built with BUILD_TESTING, BUILD_WITH_PYBIND or BUILD_WITH_CLI_TOOLS. +add_core_target(core_no_gui FALSE) - target_precompile_headers(core_no_gui - PRIVATE include/pch/pch.h - ) +target_precompile_headers(core_no_gui + PRIVATE include/pch/pch.h +) - set_target_properties(core_no_gui PROPERTIES - POSITION_INDEPENDENT_CODE ON) -endif() +set_target_properties(core_no_gui PROPERTIES + POSITION_INDEPENDENT_CODE ON) add_core_target(core TRUE) target_link_libraries(core PUBLIC core_pfd)