bin/rails mapitaine:build # rewrite the files whose records changed
bin/rails mapitaine:rebuild # rewrite everything
-
Mapitaine.build does the same from Ruby and returns what it did: written, kept, deleted, url_count, submitted and warnings. Run it on a schedule, every night or every hour. With Solid Queue:
+
Mapitaine.build does the same from Ruby and returns what it did: full, written, kept, deleted, url_count, submitted and warnings. Run it on a schedule, every night or every hour. With Solid Queue:
# config/recurring.yml
production:
sitemap:
command: "Mapitaine.build"
schedule: every hour
How a build decides
-
For each source, one query groups the records by range of ids and returns the count and the latest change of each. A range whose answer is the one the last build stored keeps its file. The others are rendered again. Nothing hooks into your models, so update_all, imports and deletions are seen too.
-
A build writes the sitemap files first, then the index, then removes the files no range fills any more, then writes sitemap-manifest.json, its memory of what each file held. A build that fails midway is repaired by the next one. Changing the host, the languages or the version of Mapitaine rewrites everything once.
+
For each source, one query groups the records by range of ids and returns the count, the sum of the ids and the latest change of each. A range whose answer is the one the last build stored keeps its file. The others are rendered again, and a file that comes out the same is not uploaded (a full build uploads every file). Nothing hooks into your models, so update_all, imports and deletions are seen too.
+
In the index, a file that changed takes the latest lastmod it holds, or the time of the build when a record left it, so that crawlers read it again.
+
Some changes leave those numbers alone: update_columns, a change in another table that the url, locales or images blocks read, a new url block, a file someone removed from the storage. Name the other table in fingerprint, or touch the record from it. For the rest, a full build comes once a week by itself:
+
config.rebuild_every = 1.day # default: a week; nil leaves it to mapitaine:rebuild
+
After deploying a change to a url block, run mapitaine:rebuild rather than wait.
+
A build notes the files it is about to add in sitemap-manifest.json, its memory of what each file held, then writes the sitemap files, then the index, then removes the files no range fills any more, then writes the manifest again. A build that fails midway is repaired by the next one. Changing the host, the languages, the size of a range or the version of Mapitaine rewrites everything it moves once.
diff --git a/lib/mapitaine/builder.rb b/lib/mapitaine/builder.rb
index 8437549..55c3743 100644
--- a/lib/mapitaine/builder.rb
+++ b/lib/mapitaine/builder.rb
@@ -2,7 +2,8 @@
module Mapitaine
class Builder
- Result = Data.define(:written, :kept, :deleted, :url_count, :submitted, :warnings) do
+ # full: whether this build rewrote everything, asked to or because rebuild_every had passed.
+ Result = Data.define(:full, :written, :kept, :deleted, :url_count, :submitted, :warnings) do
def changed?
written.any? || deleted.any?
end
@@ -14,6 +15,8 @@ def changed?
# A language, then maybe a script and a region: en, pt-BR, zh-Hant-TW, es-419.
HREFLANG = /\A[a-z]{2,3}(-[a-z]{4})?(-([a-z]{2}|\d{3}))?\z/i
+ Rendered = Data.define(:source_slug, :shard, :fingerprint, :files, :changed_urls)
+
Warning = Data.define(:source, :url, :message) do
def to_s
"#{source}: #{url} #{message}"
@@ -29,6 +32,8 @@ def initialize(configuration, storage: configuration.storage, submit: true)
def build(full: false)
ActiveSupport::Notifications.instrument("build.mapitaine", full:) do |payload|
payload[:result] = Dir.mktmpdir("mapitaine") { |directory| build_in(directory, full:) }
+ payload[:full] = payload[:result].full
+ payload[:result]
end
end
@@ -37,8 +42,11 @@ def build(full: false)
def build_in(directory, full:)
@directory = directory
@previous = Manifest.read(@storage, manifest_name)
- @reusable = !full && @previous.digest == @configuration.digest
- @manifest = Manifest.new(digest: @configuration.digest, built_at: Time.now)
+ @previous_file_by_name = @previous.files.to_h { |file| [file["name"], file] }
+ built_at = Time.now
+ @full = full || rebuild_due?(built_at)
+ @reusable = !@full && @previous.digest == @configuration.digest
+ @manifest = Manifest.new(digest: @configuration.digest, built_at:, rebuilt_at: @full ? built_at : @previous.rebuilt_at)
@written = []
@kept = []
@warnings = []
@@ -47,13 +55,16 @@ def build_in(directory, full:)
@reported = Set.new
@x_default = Xml.hreflang(@configuration.x_default)
+ @rendered = []
+
@configuration.sources.each { |source_class| build_source(source_class.new(@configuration)) }
+ publish_rendered
publish_index
- deleted = @previous.file_names - @manifest.file_names
+ deleted = (@previous.file_names | @previous.pending) - @manifest.file_names
deleted.each { |name| @storage.delete(name) }
upload(manifest_name, @manifest.to_json)
- Result.new(written: @written, kept: @kept, deleted:, url_count: @manifest.files.sum { |file| file["urls"] },
+ Result.new(full: @full, written: @written, kept: @kept, deleted:, url_count: @manifest.files.sum { |file| file["urls"] },
submitted: submit_changed_urls, warnings: @warnings)
end
@@ -64,7 +75,7 @@ def build_source(source)
if fingerprint && stored && stored["fingerprint"] == fingerprint
keep(source, shard, stored)
else
- render(source, shard, fingerprint, stored, seen_urls)
+ render(source, shard, fingerprint, seen_urls)
end
end
end
@@ -74,7 +85,7 @@ def keep(source, shard, stored)
@kept.concat(stored["files"].map { |file| file["name"] })
end
- def render(source, shard, fingerprint, stored, seen_urls)
+ def render(source, shard, fingerprint, seen_urls)
writer = ShardWriter.new(directory: @directory, basename: source.basename(shard))
changed_urls = []
ActiveSupport::Notifications.instrument("shard.mapitaine", source: source.slug, shard:) do
@@ -87,21 +98,74 @@ def render(source, shard, fingerprint, stored, seen_urls)
end
fingerprint ||= writer.content_digest
- return keep(source, shard, stored) if stored && stored["fingerprint"] == fingerprint
+ # Recorded now, published once every shard is rendered: the index keeps the order of the sources.
+ @manifest.record(source.slug, shard, fingerprint:, files: [])
+ @rendered << Rendered.new(source_slug: source.slug, shard:, fingerprint:, files: writer.files, changed_urls:)
+ end
+
+ def publish_rendered
+ changed_names = @rendered.flat_map(&:files).reject { |file| same_as_before?(file) }.to_set(&:name)
+ announce_new_files(changed_names)
+ @rendered.each do |rendered|
+ entries = rendered.files.map do |file|
+ changed = changed_names.include?(file.name)
+ publish(file, changed)
+ manifest_entry(file, changed)
+ end
+ @manifest.record(rendered.source_slug, rendered.shard, fingerprint: rendered.fingerprint, files: entries)
+ @changed_urls.concat(rendered.changed_urls) if rendered.files.any? { |file| changed_names.include?(file.name) }
+ end
+ end
+
+ def same_as_before?(file)
+ @previous_file_by_name.dig(file.name, "digest") == file.digest
+ end
- publish(source, shard, fingerprint, writer.files)
- @changed_urls.concat(changed_urls)
+ # Recorded before the first upload, so that a build dying midway leaves no file the next one cannot remove.
+ def announce_new_files(names)
+ new_names = names.to_a - @previous_file_by_name.keys - @previous.pending
+ return if new_names.empty?
+
+ @previous.pending.concat(new_names)
+ upload(manifest_name, @previous.to_json)
end
- def publish(source, shard, fingerprint, files)
- files.each do |file|
+ # A full build uploads every file, so that it also repairs a storage that lost one.
+ def publish(file, changed)
+ if changed || @full
@storage.write(file.name, file.path)
- File.delete(file.path)
+ @written << file.name
+ else
+ @kept << file.name
+ end
+ File.delete(file.path)
+ end
+
+ # The index tells crawlers which files to read again: a file that changed gets a later date even when the
+ # change is a removal, and one that did not keeps its date.
+ def manifest_entry(file, changed)
+ previous_lastmod = @previous_file_by_name.dig(file.name, "lastmod")
+ lastmod = if !changed
+ previous_lastmod
+ elsif @previous_file_by_name.key?(file.name) && !later?(file.lastmod, previous_lastmod)
+ @manifest.built_at.getutc.iso8601
+ else
+ file.lastmod
end
- @written.concat(files.map(&:name))
- @manifest.record(source.slug, shard, fingerprint:, files: files.map do |file|
- {"name" => file.name, "urls" => file.url_count, "lastmod" => file.lastmod}
- end)
+ {"name" => file.name, "urls" => file.url_count, "lastmod" => lastmod, "digest" => file.digest}
+ end
+
+ # A manifest written by an older version may hold a lastmod in another format: compare times, not text.
+ def later?(stamp, previous_stamp)
+ return false if stamp.nil?
+
+ previous_stamp = Xml.timestamp(previous_stamp)
+ previous_stamp.nil? || Xml.time(stamp) > Xml.time(previous_stamp)
+ end
+
+ def rebuild_due?(now)
+ every = @configuration.rebuild_every
+ every.present? && (@previous.rebuilt_at.nil? || @previous.rebuilt_at + every <= now)
end
def publish_index
diff --git a/lib/mapitaine/configuration.rb b/lib/mapitaine/configuration.rb
index 129656f..4703089 100644
--- a/lib/mapitaine/configuration.rb
+++ b/lib/mapitaine/configuration.rb
@@ -2,13 +2,18 @@
module Mapitaine
class Configuration
- attr_accessor :filename, :index_now_key, :index_now_key_location, :load_sources, :storage, :x_default
+ # A full build now and then catches what no fingerprint sees: update_columns, a change in another table,
+ # a new url block, a file removed from the storage.
+ DEFAULT_REBUILD_EVERY_SECONDS = 7 * 24 * 60 * 60
+
+ attr_accessor :filename, :index_now_key, :index_now_key_location, :load_sources, :rebuild_every, :storage, :x_default
attr_reader :host
attr_writer :locales, :sitemaps_url, :sources
def initialize
@filename = "sitemap"
@load_sources = -> {}
+ @rebuild_every = DEFAULT_REBUILD_EVERY_SECONDS
@storage = Storage::Local.new("public")
end
@@ -38,7 +43,7 @@ def sources
# Whatever changes every file: a build under another digest starts over.
def digest
- Digest::SHA256.hexdigest([VERSION, host!, sitemaps_url, locales, x_default, filename].to_json)
+ Digest::SHA256.hexdigest(JSON.generate([VERSION, host!, sitemaps_url, locales, x_default, filename]))
end
private
diff --git a/lib/mapitaine/index_now.rb b/lib/mapitaine/index_now.rb
index 55390d8..faba80e 100644
--- a/lib/mapitaine/index_now.rb
+++ b/lib/mapitaine/index_now.rb
@@ -19,7 +19,7 @@ def initialize(host:, key:, key_location: nil, transport: Net::HTTP.method(:post
def submit(urls)
urls.each_slice(MAX_URLS_PER_REQUEST).sum do |batch|
body = {host: @host, key: @key, keyLocation: @key_location, urlList: batch}.compact
- response = @transport.call(ENDPOINT, body.to_json, HEADERS)
+ response = @transport.call(ENDPOINT, JSON.generate(body), HEADERS)
response.code.to_i.between?(200, 299) ? batch.size : 0
end
end
diff --git a/lib/mapitaine/manifest.rb b/lib/mapitaine/manifest.rb
index d85cf4d..da45644 100644
--- a/lib/mapitaine/manifest.rb
+++ b/lib/mapitaine/manifest.rb
@@ -1,25 +1,33 @@
# frozen_string_literal: true
module Mapitaine
- # What the last build wrote: for each shard of each source, its fingerprint and its files.
+ # What the last build wrote: for each shard of each source, its fingerprint and its files. pending lists the
+ # files a build started to publish before it could record them, so that the next one can remove them.
class Manifest
- attr_reader :built_at, :digest, :shards_by_source
+ attr_reader :built_at, :rebuilt_at, :digest, :shards_by_source, :pending
def self.read(storage, name)
json = storage.read(name)
return new unless json
stored = JSON.parse(json)
- new(digest: stored["digest"], built_at: Time.iso8601(stored["built_at"]), shards_by_source: stored["sources"])
+ new(digest: stored["digest"], built_at: time(stored["built_at"]), rebuilt_at: time(stored["rebuilt_at"]),
+ shards_by_source: stored["sources"] || {}, pending: stored["pending"] || [])
rescue JSON::ParserError, ArgumentError, TypeError
# A manifest we cannot read is a manifest we do not have: the build starts over and replaces it.
new
end
- def initialize(digest: nil, built_at: nil, shards_by_source: {})
+ def self.time(text)
+ Time.iso8601(text) if text
+ end
+
+ def initialize(digest: nil, built_at: nil, rebuilt_at: nil, shards_by_source: {}, pending: [])
@digest = digest
@built_at = built_at
+ @rebuilt_at = rebuilt_at
@shards_by_source = shards_by_source
+ @pending = pending
end
def shard(source, key)
@@ -39,7 +47,10 @@ def file_names
end
def to_json(*)
- JSON.pretty_generate("digest" => digest, "built_at" => built_at.utc.iso8601, "sources" => shards_by_source)
+ # Whole seconds, like the lastmods compared to built_at: a change in the second of a build is sent again
+ # rather than never.
+ JSON.pretty_generate("digest" => digest, "built_at" => built_at&.utc&.iso8601,
+ "rebuilt_at" => rebuilt_at&.utc&.iso8601, "pending" => pending, "sources" => shards_by_source)
end
end
end
diff --git a/lib/mapitaine/shard_writer.rb b/lib/mapitaine/shard_writer.rb
index da387f4..74558fd 100644
--- a/lib/mapitaine/shard_writer.rb
+++ b/lib/mapitaine/shard_writer.rb
@@ -7,7 +7,7 @@ class ShardWriter
MAX_URLS_PER_FILE = 50_000
MAX_BYTES_PER_FILE = 52_428_800
- Written = Data.define(:name, :path, :url_count, :lastmod)
+ Written = Data.define(:name, :path, :url_count, :lastmod, :digest)
attr_reader :files
@@ -17,7 +17,6 @@ def initialize(directory:, basename:, max_urls: MAX_URLS_PER_FILE, max_bytes: MA
@max_urls = max_urls
@max_bytes = max_bytes
@files = []
- @digest = Digest::SHA256.new
end
# The entries of one page stay in the same file.
@@ -36,7 +35,7 @@ def close
end
def content_digest
- @digest.hexdigest
+ Digest::SHA256.hexdigest(@files.map(&:digest).join)
end
private
@@ -46,6 +45,7 @@ def open_file
@name = "#{@basename}#{suffix}.xml.gz"
@path = File.join(@directory, @name)
@gzip = Zlib::GzipWriter.open(@path)
+ @file_digest = Digest::SHA256.new
@url_count = 0
@bytes = Xml::URLSET_CLOSE.bytesize
@lastmod = nil
@@ -53,16 +53,15 @@ def open_file
end
def roll
- @gzip.write(Xml::URLSET_CLOSE)
- @digest << Xml::URLSET_CLOSE
+ write(Xml::URLSET_CLOSE)
@gzip.close
@gzip = nil
- @files << Written.new(name: @name, path: @path, url_count: @url_count, lastmod: @lastmod)
+ @files << Written.new(name: @name, path: @path, url_count: @url_count, lastmod: @lastmod, digest: @file_digest.hexdigest)
end
def write(text)
@gzip.write(text)
- @digest << text
+ @file_digest << text
@bytes += text.bytesize
end
end
diff --git a/lib/mapitaine/source.rb b/lib/mapitaine/source.rb
index 87c3365..1ba4cd6 100644
--- a/lib/mapitaine/source.rb
+++ b/lib/mapitaine/source.rb
@@ -102,8 +102,11 @@ def default_url_options
def fingerprint_by_shard
return {WHOLE => nil} unless relation?
- relation.reorder(nil).group(shard_sql).pluck(shard_sql, Arel.sql("COUNT(*)"), fingerprint_sql)
- .to_h { |shard, count, latest| [shard.to_i.to_s, "#{count}/#{precise(latest)}"] }
+ # The sum of the ids moves when a record leaves a range and another enters it; the shard size, when the
+ # ranges themselves move. Summed as offsets within the range: SQLite's SUM overflows on large ids.
+ offset_sum = Arel.sql("SUM(#{model.quoted_table_name}.#{model.quoted_primary_key} % #{shard_size})")
+ relation.reorder(nil).group(shard_sql).pluck(shard_sql, Arel.sql("COUNT(*)"), offset_sum, fingerprint_sql)
+ .to_h { |shard, count, sum, latest| [shard.to_i.to_s, "#{shard_size}/#{count}/#{sum.to_i}/#{precise(latest)}"] }
end
def each_page(shard)
diff --git a/lib/mapitaine/tasks.rake b/lib/mapitaine/tasks.rake
index 1fe25cf..8b680b3 100644
--- a/lib/mapitaine/tasks.rake
+++ b/lib/mapitaine/tasks.rake
@@ -2,7 +2,7 @@
namespace :mapitaine do
report = lambda do |result|
- puts "#{result.url_count} URLs: #{result.written.size} files written, #{result.kept.size} kept, " \
+ puts "#{result.url_count} URLs#{" (full build)" if result.full}: #{result.written.size} files written, #{result.kept.size} kept, " \
"#{result.deleted.size} deleted, #{result.submitted} URLs sent to IndexNow"
result.warnings.each { |warning| warn " #{warning}" }
end
diff --git a/test/build_test.rb b/test/build_test.rb
index ea4f82f..73ccc9d 100644
--- a/test/build_test.rb
+++ b/test/build_test.rb
@@ -3,13 +3,13 @@
require "test_helper"
class BuildTest < MapitaineTest
- def test_first_build_writes_a_file_per_shard_then_the_index_then_the_manifest
+ def test_first_build_announces_its_files_writes_one_per_shard_then_the_index_then_the_manifest
create_post(1)
create_post(12)
built = build
- assert_equal %w[sitemap-post-00000.xml.gz sitemap-post-00001.xml.gz sitemap-page.xml.gz sitemap.xml sitemap-manifest.json],
- storage.writes
+ assert_equal %w[sitemap-manifest.json sitemap-post-00000.xml.gz sitemap-post-00001.xml.gz sitemap-page.xml.gz sitemap.xml
+ sitemap-manifest.json], storage.writes
assert_equal %w[sitemap-post-00000.xml.gz sitemap-post-00001.xml.gz sitemap-page.xml.gz], built.written
assert_equal 6, built.url_count
assert_empty built.warnings
@@ -108,6 +108,135 @@ def test_a_shard_left_without_records_loses_its_file
assert_predicate built, :changed?
end
+ def test_a_record_replaced_by_an_older_one_rewrites_its_shard
+ create_post(1, updated_at: LONG_AGO + 60)
+ create_post(2)
+ build
+ Post.where(id: 2).delete_all
+ create_post(3)
+ built = build
+
+ assert_equal %w[sitemap-post-00000.xml.gz], built.written
+ assert_includes storage.xml("sitemap-post-00000.xml.gz"), "posts/3"
+ end
+
+ def test_ids_too_large_for_a_sum_are_fingerprinted
+ create_post(9_000_000_000_000_000_001)
+ create_post(9_000_000_000_000_000_002)
+ build
+ Post.where(id: 9_000_000_000_000_000_002).delete_all
+ create_post(9_000_000_000_000_000_003)
+
+ assert_equal %w[sitemap-post-900000000000000000.xml.gz], build.written
+ end
+
+ def test_a_new_shard_size_rewrites_the_source
+ create_post(5)
+ create_post(15)
+ build
+ Post.where(id: 15).delete_all
+ create_post(25)
+ Mapitaine.configuration.sources = [Class.new(PostSitemap) { shard_size 20 }, PageSitemap]
+ built = build
+
+ assert_equal %w[sitemap-post-00001.xml.gz], built.written
+ assert_includes built.kept, "sitemap-post-00000.xml.gz"
+ refute_includes storage.xml("sitemap-post-00001.xml.gz"), "posts/15"
+ assert_includes storage.xml("sitemap-post-00001.xml.gz"), "posts/25"
+ end
+
+ def test_a_file_that_changed_moves_forward_in_the_index_even_when_a_record_left_it
+ create_post(1)
+ create_post(2)
+ build
+ Post.where(id: 2).delete_all
+ built = build
+
+ assert_equal %w[sitemap-post-00000.xml.gz], built.written
+ stamp = storage.xml("sitemap.xml")[%r{sitemap-post-00000.xml.gz