From 2474f52e9b2ad2b01a9889690a394c515dd289ad Mon Sep 17 00:00:00 2001 From: eyal rozen Date: Wed, 16 Sep 2026 17:48:01 +0300 Subject: [PATCH 1/4] feat: create history tables for tables with history enabled Adds the schema half of feature history. A table definition may now set history = true, which makes osm2pgsql: - add a "valid_from timestamptz DEFAULT now()" column to the table. It is create_only, so it never appears in the COPY column list and the database default stamps it on every insert. The config cannot declare valid_from or valid_to itself; doing so is an error, so a table with history can never end up without a validity interval. - create _history, mirroring the table column for column plus "valid_to", which closes the interval of a superseded version. Mirroring exactly lets the rows deleted from the live table be inserted without naming columns. The history table is created IF NOT EXISTS and is never dropped, including in create mode, so history outlives a re-import. Moving superseded rows into it, and the create-mode schema check, follow in later commits. Refs: MAPCO-11663 --- src/flex-lua-table.cpp | 32 ++++++++++++++++++++++++++++++++ src/flex-table.cpp | 30 ++++++++++++++++++++++++++++++ src/flex-table.hpp | 11 +++++++++++ 3 files changed, 73 insertions(+) diff --git a/src/flex-lua-table.cpp b/src/flex-lua-table.cpp index b88c58261..d99ca4561 100644 --- a/src/flex-lua-table.cpp +++ b/src/flex-lua-table.cpp @@ -446,6 +446,37 @@ TRAMPOLINE_WRAPPED_OBJECT(table, schema) } // anonymous namespace +void setup_flex_table_history(lua_State *lua_state, flex_table_t *table) +{ + assert(lua_state); + assert(table); + + lua_getfield(lua_state, -1, "history"); + if (lua_isboolean(lua_state, -1)) { + table->set_has_history(lua_toboolean(lua_state, -1)); + } else if (!lua_isnil(lua_state, -1)) { + throw fmt_error("The 'history' field in table '{}' must be a boolean.", + table->name()); + } + lua_pop(lua_state, 1); + + if (!table->has_history()) { + return; + } + + for (char const *name : {"valid_from", "valid_to"}) { + if (util::find_by_name(table->columns(), name)) { + throw fmt_error( + "Table '{}' has history enabled, so column '{}' is added by" + " osm2pgsql and must not be defined in the Lua config.", + table->name(), name); + } + } + + table->add_column("valid_from", "timestamptz", "timestamptz DEFAULT now()") + .set_create_only(); +} + int setup_flex_table(lua_State *lua_state, std::vector *tables, std::vector *expire_outputs, std::string const &default_schema, bool updatable, @@ -461,6 +492,7 @@ int setup_flex_table(lua_State *lua_state, std::vector *tables, setup_flex_table_columns(lua_state, &new_table, expire_outputs, append_mode); setup_flex_table_indexes(lua_state, &new_table, updatable); + setup_flex_table_history(lua_state, &new_table); void *ptr = lua_newuserdata(lua_state, sizeof(std::size_t)); auto *num = new (ptr) std::size_t{}; diff --git a/src/flex-table.cpp b/src/flex-table.cpp index 263a9cedc..98682168f 100644 --- a/src/flex-table.cpp +++ b/src/flex-table.cpp @@ -60,6 +60,11 @@ std::string flex_table_t::full_name() const return qualified_name(schema(), name()); } +std::string flex_table_t::full_history_name() const +{ + return qualified_name(schema(), history_name()); +} + std::string flex_table_t::full_tmp_name() const { return qualified_name(schema(), name() + "_tmp"); @@ -207,6 +212,27 @@ flex_table_t::build_sql_create_table(table_type ttype, return sql; } +std::string flex_table_t::build_sql_create_history_table() const +{ + assert(!m_columns.empty()); + assert(m_has_history); + + std::string sql = + fmt::format("CREATE TABLE IF NOT EXISTS {} (", full_history_name()); + + util::string_joiner_t joiner{','}; + for (auto const &column : m_columns) { + joiner.add(column.sql_create()); + } + joiner.add(R"("valid_to" timestamptz)"); + + sql += joiner(); + sql += ')'; + sql += tablespace_clause(m_data_tablespace); + + return sql; +} + std::string flex_table_t::build_sql_column_list() const { assert(!m_columns.empty()); @@ -315,6 +341,10 @@ void table_connection_t::start(pg_conn_t const &db_connection, enable_check_trigger(db_connection, table()); } + if (table().has_history()) { + db_connection.exec(table().build_sql_create_history_table()); + } + table().prepare(db_connection); } diff --git a/src/flex-table.hpp b/src/flex-table.hpp index 663375327..c18bde6d0 100644 --- a/src/flex-table.hpp +++ b/src/flex-table.hpp @@ -174,6 +174,15 @@ class flex_table_t std::string full_name() const; std::string full_tmp_name() const; + bool has_history() const noexcept { return m_has_history; } + + void set_has_history(bool value) noexcept { m_has_history = value; } + + std::string history_name() const { return m_name + "_history"; } + std::string full_history_name() const; + + std::string build_sql_create_history_table() const; + bool has_multiple_geom_columns() const noexcept { return m_has_multiple_geom_columns; @@ -266,6 +275,8 @@ class flex_table_t /// Does this table have more than one geometry column? bool m_has_multiple_geom_columns = false; + bool m_has_history = false; + /// Always build the id index, not only when it is needed for updates? bool m_always_build_id_index = false; From 818db2ad8467c6c6e74d6d31180b7f9871e57000 Mon Sep 17 00:00:00 2001 From: eyal rozen Date: Wed, 16 Sep 2026 18:02:55 +0300 Subject: [PATCH 2/4] feat: move superseded rows into the history table Row replacement in append mode is a plain DELETE, so the old version of every modified or deleted feature was discarded. For a table with history enabled that DELETE becomes a data-modifying CTE which inserts the deleted rows into
_history with now() as valid_to, closing their validity interval. Both deleters are covered: the by-id form and the by-type-and-id form, whose USING (VALUES ...) join needs RETURNING p.* rather than a bare * so the VALUES columns are not returned alongside the row. The history table is an exact column-for-column mirror of the live table, so the insert needs no column list and gfid, the geometry and every attribute column ride along unchanged. Tables without history are unaffected: the generated SQL is identical to before. Stage-2 and pending-object reprocessing also delete and re-insert, which produces empty [now, now) intervals. Those never match a point-in-time query, so they need no filtering. Refs: MAPCO-11663 --- src/db-copy.cpp | 35 +++++++++++++++++++++++++++++++++-- src/db-copy.hpp | 8 +++++++- src/flex-table.hpp | 3 +++ 3 files changed, 43 insertions(+), 3 deletions(-) diff --git a/src/db-copy.cpp b/src/db-copy.cpp index 1a7dedec5..08b5a00fd 100644 --- a/src/db-copy.cpp +++ b/src/db-copy.cpp @@ -20,13 +20,18 @@ void db_deleter_by_id_t::delete_rows(std::string const &table, std::string const &column, + std::string const &history, pg_conn_t const &db_connection) { fmt::memory_buffer sql; // Each deletable contributes an OSM ID and a comma. The highest node ID // currently has 10 digits, so 15 characters should do for a couple of years. // Add 50 characters for the SQL statement itself. - sql.reserve(m_deletables.size() * 15 + 50); + sql.reserve(m_deletables.size() * 15 + 50 + history.size() * 2); + + if (!history.empty()) { + fmt::format_to(std::back_inserter(sql), FMT_STRING("WITH d AS (")); + } fmt::format_to(std::back_inserter(sql), FMT_STRING("DELETE FROM {} WHERE {} IN ("), table, column); @@ -36,12 +41,20 @@ void db_deleter_by_id_t::delete_rows(std::string const &table, } sql[sql.size() - 1] = ')'; + if (!history.empty()) { + fmt::format_to( + std::back_inserter(sql), + FMT_STRING(" RETURNING *) INSERT INTO {} SELECT d.*, now() FROM d"), + history); + } + sql.push_back('\0'); db_connection.exec(sql.data()); } void db_deleter_by_type_and_id_t::delete_rows(std::string const &table, std::string const &column, + std::string const &history, pg_conn_t const &db_connection) { assert(!m_deletables.empty()); @@ -50,7 +63,11 @@ void db_deleter_by_type_and_id_t::delete_rows(std::string const &table, // Need a VALUES line for each deletable: type (3 bytes), id (15 bytes), // braces etc. (4 bytes). And additional space for the remainder of the // SQL command. - sql.reserve(m_deletables.size() * 22 + 200); + sql.reserve(m_deletables.size() * 22 + 200 + history.size() * 2); + + if (!history.empty()) { + fmt::format_to(std::back_inserter(sql), FMT_STRING("WITH d AS (")); + } if (m_has_type) { fmt::format_to(std::back_inserter(sql), @@ -71,6 +88,13 @@ void db_deleter_by_type_and_id_t::delete_rows(std::string const &table, ") AS t (osm_type, osm_id) WHERE" " p.{} = t.osm_type::char(1) AND p.{} = t.osm_id", type, column.c_str() + pos + 1); + + if (!history.empty()) { + fmt::format_to(std::back_inserter(sql), + FMT_STRING(" RETURNING p.*) INSERT INTO {}" + " SELECT d.*, now() FROM d"), + history); + } } else { fmt::format_to(std::back_inserter(sql), FMT_STRING("DELETE FROM {} WHERE {} IN ("), table, @@ -80,6 +104,13 @@ void db_deleter_by_type_and_id_t::delete_rows(std::string const &table, format_to(std::back_inserter(sql), FMT_STRING("{},"), item.osm_id); } sql[sql.size() - 1] = ')'; + + if (!history.empty()) { + fmt::format_to(std::back_inserter(sql), + FMT_STRING(" RETURNING *) INSERT INTO {}" + " SELECT d.*, now() FROM d"), + history); + } } sql.push_back('\0'); diff --git a/src/db-copy.hpp b/src/db-copy.hpp index 3a9d3f05e..0f375d7cf 100644 --- a/src/db-copy.hpp +++ b/src/db-copy.hpp @@ -46,8 +46,10 @@ class db_target_descr_t std::string const &name() const noexcept { return m_name; } std::string const &id() const noexcept { return m_id; } std::string const &rows() const noexcept { return m_rows; } + std::string const &history() const noexcept { return m_history; } void set_rows(std::string rows) { m_rows = std::move(rows); } + void set_history(std::string history) { m_history = std::move(history); } /** * Check if the buffer would use exactly the same copy operation. @@ -68,6 +70,8 @@ class db_target_descr_t std::string m_id; /// Comma-separated list of rows for copy operation (when empty: all rows) std::string m_rows; + /// Qualified name of the history table (when empty: history disabled) + std::string m_history; }; /** @@ -87,6 +91,7 @@ class db_deleter_by_id_t void add(osmid_t osm_id) { m_deletables.push_back(osm_id); } void delete_rows(std::string const &table, std::string const &column, + std::string const &history, pg_conn_t const &db_connection); bool is_full() const noexcept { return m_deletables.size() > MAX_ENTRIES; } @@ -127,6 +132,7 @@ class db_deleter_by_type_and_id_t } void delete_rows(std::string const &table, std::string const &column, + std::string const &history, pg_conn_t const &db_connection); bool is_full() const noexcept { return m_deletables.size() > MAX_ENTRIES; } @@ -192,7 +198,7 @@ class db_cmd_copy_delete_t : public db_cmd_copy_t if (m_deleter.has_data()) { m_deleter.delete_rows( qualified_name(target->schema(), target->name()), target->id(), - db_connection); + target->history(), db_connection); } } diff --git a/src/flex-table.hpp b/src/flex-table.hpp index c18bde6d0..9c1166eb0 100644 --- a/src/flex-table.hpp +++ b/src/flex-table.hpp @@ -302,6 +302,9 @@ class table_connection_t table->build_sql_column_list())), m_copy_mgr(copy_thread) { + if (table->has_history()) { + m_target->set_history(table->full_history_name()); + } } void start(pg_conn_t const &db_connection, bool append) const; From 1b66aa4ac0911c6c937da4ea2af1c35a1f06960a Mon Sep 17 00:00:00 2001 From: eyal rozen Date: Thu, 17 Sep 2026 09:56:27 +0300 Subject: [PATCH 3/4] fix: create the history table only in create mode The history table was created in append mode too, which looks convenient but cannot work: valid_from is only added to the live table when that table is created, so on a database predating history the live table has n columns while the history table has n+2. DELETE ... RETURNING * then yields n values for an n+1 column insert and every modified feature fails, at run time, after a clean startup. History is established by a create, as ADR-0003 has it. Enabling it on an existing database needs a re-import, or an ALTER TABLE adding valid_from before the next run. Verifying this in append mode belongs with the schema check, which is still to come. Refs: MAPCO-11663 --- src/flex-table.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/flex-table.cpp b/src/flex-table.cpp index 98682168f..2e94a8898 100644 --- a/src/flex-table.cpp +++ b/src/flex-table.cpp @@ -339,10 +339,10 @@ void table_connection_t::start(pg_conn_t const &db_connection, table().full_name())); enable_check_trigger(db_connection, table()); - } - if (table().has_history()) { - db_connection.exec(table().build_sql_create_history_table()); + if (table().has_history()) { + db_connection.exec(table().build_sql_create_history_table()); + } } table().prepare(db_connection); From 76f259f35c6a86031761bf7bb4ace4e3812582b8 Mon Sep 17 00:00:00 2001 From: eyal rozen Date: Thu, 17 Sep 2026 14:14:51 +0300 Subject: [PATCH 4/4] feat: drop history versions identical to the row that replaced them MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Append mode replaces a row by deleting and re-inserting it, and the delete runs before the copy (db-copy.cpp:213 then :216), so the history CTE cannot tell a real edit from a rewrite of identical content. Two things produce the latter: re-ingesting a diff that was already applied, and node changes, which mark every way containing the node as pending and rewrite it unchanged. Both leave a history version that records no change. After the copy has synced, and only then, each superseded row this run stamped is compared against the row now live and dropped when every written column matches. Scoped by valid_to to the current run's stamp rather than the whole table: on the buildings table that is 20 rows to consider instead of 49231, so the cost follows the change, not the corpus. create_only columns are excluded from the comparison — osm2pgsql never writes them, so valid_from and a derived geom3d cannot count as a difference. Refs: MAPCO-11663 --- src/flex-table.cpp | 42 ++++++++++++++++++++++++++++++++++++++++++ src/flex-table.hpp | 2 ++ 2 files changed, 44 insertions(+) diff --git a/src/flex-table.cpp b/src/flex-table.cpp index 2e94a8898..a4871eb40 100644 --- a/src/flex-table.cpp +++ b/src/flex-table.cpp @@ -233,6 +233,42 @@ std::string flex_table_t::build_sql_create_history_table() const return sql; } +std::string flex_table_t::build_sql_dedup_history() const +{ + assert(m_has_history); + + std::string match; + util::string_joiner_t hist{','}; + util::string_joiner_t live{','}; + + for (auto const &column : m_columns) { + if (column.type() == table_column_type::id_type || + column.type() == table_column_type::id_num) { + if (!match.empty()) { + match += " AND "; + } + match += fmt::format(R"(h."{0}" = b."{0}")", column.name()); + continue; + } + if (column.create_only()) { + continue; + } + hist.add(fmt::format(R"(h."{}")", column.name())); + live.add(fmt::format(R"(b."{}")", column.name())); + } + + if (match.empty() || hist.empty()) { + return {}; + } + + return fmt::format( + R"(DELETE FROM {} h USING {} b WHERE {})" + R"( AND h."valid_to" = (SELECT max("valid_to") FROM {}))" + R"( AND ROW({}) IS NOT DISTINCT FROM ROW({}))", + full_history_name(), full_name(), match, full_history_name(), hist(), + live()); +} + std::string flex_table_t::build_sql_column_list() const { assert(!m_columns.empty()); @@ -354,6 +390,12 @@ void table_connection_t::stop(pg_conn_t const &db_connection, bool updateable, m_copy_mgr.sync(); if (append) { + if (table().has_history() && table().has_id_column()) { + auto const sql = table().build_sql_dedup_history(); + if (!sql.empty()) { + db_connection.exec(sql); + } + } return; } diff --git a/src/flex-table.hpp b/src/flex-table.hpp index 9c1166eb0..d72446962 100644 --- a/src/flex-table.hpp +++ b/src/flex-table.hpp @@ -183,6 +183,8 @@ class flex_table_t std::string build_sql_create_history_table() const; + std::string build_sql_dedup_history() const; + bool has_multiple_geom_columns() const noexcept { return m_has_multiple_geom_columns;