From f13c454d5fc571d26149202441d67ba0642fb3df Mon Sep 17 00:00:00 2001 From: Robert Curth Date: Sat, 7 Nov 2026 15:50:47 +0100 Subject: [PATCH 1/4] Share the model setup of the release backfills and publishing Pull the Ollama check of backfill_suggestions into each_site_with_model, the default of the model setting into Ollama's setting/2, and the transaction around publish_stored/3 that Saving and Backfill both open into Versions.publish_committed/3, so that the embeddings can hook into them. No change in behaviour. Refs #522 Co-Authored-By: Claude Opus 5.5 --- lib/feather/content/backfill.ex | 10 ++++------ lib/feather/content/saving.ex | 8 +++----- lib/feather/content/versions.ex | 5 +++++ lib/feather/ollama.ex | 10 ++++++---- lib/feather/release.ex | 28 ++++++++++++++++------------ 5 files changed, 34 insertions(+), 27 deletions(-) diff --git a/lib/feather/content/backfill.ex b/lib/feather/content/backfill.ex index 2c1289cc..0e41bc10 100644 --- a/lib/feather/content/backfill.ex +++ b/lib/feather/content/backfill.ex @@ -46,12 +46,10 @@ defmodule Feather.Content.Backfill do if suggestions == %{} do :unchanged else - Repo.transact(fn -> - Versions.publish_stored(scope, record, - state: :published, - prepare: &Suggestions.put_suggestions(&1, suggestions) - ) - end) + Versions.publish_committed(scope, record, + state: :published, + prepare: &Suggestions.put_suggestions(&1, suggestions) + ) end end end diff --git a/lib/feather/content/saving.ex b/lib/feather/content/saving.ex index db4e911a..16797f41 100644 --- a/lib/feather/content/saving.ex +++ b/lib/feather/content/saving.ex @@ -52,11 +52,9 @@ defmodule Feather.Content.Saving do {:error, :stale} -> %{} end - Repo.transact(fn -> - Versions.publish_stored(scope, record, - prepare: &Suggestions.put_suggestions(&1, suggestions) - ) - end) + Versions.publish_committed(scope, record, + prepare: &Suggestions.put_suggestions(&1, suggestions) + ) end @doc false diff --git a/lib/feather/content/versions.ex b/lib/feather/content/versions.ex index eb7ae42a..e8eaa785 100644 --- a/lib/feather/content/versions.ex +++ b/lib/feather/content/versions.ex @@ -43,6 +43,11 @@ defmodule Feather.Content.Versions do @doc false def lock_opts, do: @lock_opts + @doc false + # publish_stored/3 in its own transaction. + def publish_committed(scope, record, opts \\ []), + do: Repo.transact(fn -> publish_stored(scope, record, opts) end) + @doc false # Publishes the stored record in the caller's transaction. `state: :unpublished` (the # default) publishes a draft or unpublished changes and leaves a published record as it diff --git a/lib/feather/ollama.ex b/lib/feather/ollama.ex index 8435381f..e7297373 100644 --- a/lib/feather/ollama.ex +++ b/lib/feather/ollama.ex @@ -199,10 +199,12 @@ defmodule Feather.Ollama do defp url, do: Application.get_env(:feather, :ollama_url) - defp model do - case Application.get_env(:feather, :ollama_model) do - model when model in [nil, ""] -> @default_model - model -> model + defp model, do: setting(:ollama_model, @default_model) + + defp setting(key, default) do + case Application.get_env(:feather, key) do + value when value in [nil, ""] -> default + value -> value end end end diff --git a/lib/feather/release.ex b/lib/feather/release.ex index 6c2d2a1f..e858f6b8 100644 --- a/lib/feather/release.ex +++ b/lib/feather/release.ex @@ -110,18 +110,7 @@ defmodule Feather.Release do changes, by publishing them again (`Feather.Content.backfill_suggestions/1`). Needs `OLLAMA_URL`. Deploy the sites by hand afterwards. """ - def backfill_suggestions do - with_app(fn -> - {:ok, _} = Application.ensure_all_started(:req) - - if Feather.Ollama.configured?() do - Enum.each(Feather.Repo.all(Site), &backfill_site/1) - else - IO.puts(:stderr, "OLLAMA_URL is not set.") - {:error, :not_configured} - end - end) - end + def backfill_suggestions, do: each_site_with_model(&backfill_site/1) defp backfill_site(site) do scope = Scope.for_site(site) @@ -133,6 +122,21 @@ defmodule Feather.Release do ) end + defp each_site_with_model(fun) do + with_app(fn -> + {:ok, _} = Application.ensure_all_started(:req) + + if Feather.Ollama.configured?(), + do: Enum.each(Feather.Repo.all(Site), fun), + else: not_configured() + end) + end + + defp not_configured do + IO.puts(:stderr, "OLLAMA_URL is not set.") + {:error, :not_configured} + end + defp with_app(fun) do load_app() {:ok, _} = Application.ensure_all_started(:ecto_sql) From 92139eb8f340ec0c3638049048b771761ed18505 Mon Sep 17 00:00:00 2001 From: Robert Curth Date: Sat, 7 Nov 2026 15:51:03 +0100 Subject: [PATCH 2/4] Read published posts with the embedding model After a post is published, EmbeddingGemma 2 (OLLAMA_EMBEDDING_MODEL) reads its published version in the background through Ollama's embed API: the clustering prompt with title and description, then the first 4000 characters of its plain text. The L2-normalised vector is stored as 32-bit floats with the model's name on the post version, and storing it for a post already live counts the site as changed. backfill_embeddings reads the published versions without a vector of the configured model, publishing nothing again; the release backfills read inline, since eval runs no background tasks. This is what the related posts are found by. Refs #522 Co-Authored-By: Claude Opus 5.5 --- config/runtime.exs | 4 + config/test.exs | 2 + lib/feather/content.ex | 3 +- lib/feather/content/embedding.ex | 152 ++++++++ lib/feather/content/post_version.ex | 5 + lib/feather/content/saving.ex | 4 +- lib/feather/content/versions.ex | 12 +- lib/feather/ollama.ex | 42 ++- lib/feather/release.ex | 42 ++- ...7113721_add_embedding_to_post_versions.exs | 10 + test/feather/content/embedding_test.exs | 343 ++++++++++++++++++ test/feather/ollama_test.exs | 70 +++- test/feather/release_test.exs | 61 +++- test/support/fixtures/content_fixtures.ex | 19 +- 14 files changed, 741 insertions(+), 28 deletions(-) create mode 100644 lib/feather/content/embedding.ex create mode 100644 priv/repo/migrations/20261107113721_add_embedding_to_post_versions.exs create mode 100644 test/feather/content/embedding_test.exs diff --git a/config/runtime.exs b/config/runtime.exs index fda4aa4b..a71cb3dc 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -25,6 +25,10 @@ if config_env() != :test do config :feather, :ollama_model, ollama_model end + if ollama_embedding_model = System.get_env("OLLAMA_EMBEDDING_MODEL") do + config :feather, :ollama_embedding_model, ollama_embedding_model + end + case System.get_env("FEDIVERSE_PROXY", "") do "" -> :ok diff --git a/config/test.exs b/config/test.exs index 7a9464d6..7b7c73a5 100644 --- a/config/test.exs +++ b/config/test.exs @@ -24,6 +24,8 @@ config :feather, backfill_image_variants: false, image_variant_sync: :inline, alt_text_suggestions: :inline, + # Publishing reads no post for its related posts unless a test asks for it (:inline). + post_embeddings: false, fediverse_deliveries: :inline, fediverse_inbox: :inline, # Deploys tell IndexNow and archive.org nothing unless a test asks for it. diff --git a/lib/feather/content.ex b/lib/feather/content.ex index ac379872..5f9c73f0 100644 --- a/lib/feather/content.ex +++ b/lib/feather/content.ex @@ -35,7 +35,7 @@ defmodule Feather.Content do alias Feather.Accounts.Scope alias Feather.Books - alias Feather.Content.{Backfill, Blocks, Page, Pages, Post, Posts, Project, Projects} + alias Feather.Content.{Backfill, Blocks, Embedding, Page, Pages, Post, Posts, Project, Projects} alias Feather.Content.{ProseMirror, Saving, Slug, Suggestions, Sync, Tags, Versions} alias Feather.Repo alias Feather.Sites.Site @@ -92,6 +92,7 @@ defmodule Feather.Content do defdelegate publish(scope, record), to: Saving defdelegate unpublish(scope, record), to: Versions defdelegate backfill_suggestions(scope), to: Backfill + defdelegate backfill_embeddings(scope), to: Embedding, as: :backfill defdelegate suggest_description(scope, record), to: Suggestions defdelegate put_suggested_description(scope, record, description), to: Suggestions defdelegate published_slug?(scope, schema, slug, except_id \\ nil), to: Versions diff --git a/lib/feather/content/embedding.ex b/lib/feather/content/embedding.ex new file mode 100644 index 00000000..1dc9edd5 --- /dev/null +++ b/lib/feather/content/embedding.ex @@ -0,0 +1,152 @@ +defmodule Feather.Content.Embedding do + @moduledoc """ + What the embedding model of `Feather.Ollama` reads in a published post + (ADR 0036), which its related posts are found by + (`Feather.Content.RelatedPosts`). + + The model reads the clustering prompt with the title (a short post's + meta title, `none` without either) and the description, then the first + 4000 characters of the plain text. + + Publishing a post reads its new version in the background (`start/1`); + `backfill/1` reads the published versions read by no or another model. + The vector, which Ollama returns L2-normalised, is stored as + little-endian 32-bit floats on the version, with the model's name, so that only vectors of the configured + model are compared (`store/3`). Without the model, and for a version + without plain text, the version stays unread. + """ + + import Ecto.Query, warn: false + + alias Feather.Accounts.Scope + alias Feather.Content.{Blocks, Post, PostVersion} + alias Feather.{Ollama, Publishing, Repo, Sites} + alias Feather.Sites.Site + alias Feather.Validations + + require Logger + + @text_length 4000 + + @doc """ + Reads the published version of a post under `Feather.TaskSupervisor`, + or right away with `config :feather, :post_embeddings, :inline`; `false` + reads nothing (tests that do not ask for it). Does nothing without + `OLLAMA_URL` or when the version is read by the configured model. + """ + @spec start(Post.t()) :: :ok + def start(%Post{published_version: %PostVersion{} = version}) do + if Ollama.configured?() and not read?(version), + do: start_in(Application.get_env(:feather, :post_embeddings, :background), version), + else: :ok + end + + def start(%Post{}), do: :ok + + defp start_in(:background, version) do + Feather.TaskSupervisor + |> Task.Supervisor.start_child(fn -> embed(version) end) + |> log_start_failure(version) + end + + defp start_in(:inline, version) do + embed(version) + :ok + end + + defp start_in(false, _version), do: :ok + + defp log_start_failure({:ok, _pid}, _version), do: :ok + + defp log_start_failure({:error, reason}, version) do + Logger.error("Post version #{version.id} not read: #{inspect(reason)}") + end + + defp read?(%PostVersion{embedding: embedding, embedding_model: model}), + do: is_binary(embedding) and model == Ollama.embedding_model() + + defp embed(%PostVersion{} = version) do + with {:ok, input} <- input(version), + {:ok, vector} <- Ollama.embedding(input) do + store(version, vector, Ollama.embedding_model()) + else + {:error, _reason} -> :error + end + end + + @doc """ + Stores the vector, as Ollama returns it L2-normalised, as the version's + embedding by `model`. When the version is the published version + of a post the live site already shows, the site counts as changed since + its last deploy, as the related posts the export shows may change. + """ + @spec store(PostVersion.t(), [number()], String.t()) :: :ok + def store(%PostVersion{id: id} = version, vector, model) do + Repo.update_all(from(v in PostVersion, where: v.id == ^id), + set: [embedding: encode(vector), embedding_model: model] + ) + + touch_site_if_live(version) + end + + defp touch_site_if_live(%PostVersion{id: id} = version) do + case Repo.one(from p in Post, where: p.published_version_id == ^id, select: p.site_id) do + nil -> :ok + site_id -> touch_site_if_live(Scope.for_site(%Site{id: site_id}), version) + end + end + + defp touch_site_if_live(scope, version) do + target = Publishing.get_target(scope) + + if target && Publishing.live?(target, version), + do: Sites.touch_site(scope), + else: :ok + end + + defp input(%PostVersion{} = version) do + case Blocks.model_text(version.content, @text_length) do + "" -> + {:error, :no_text} + + text -> + {:ok, "task: clustering | query: #{title(version)}. #{version.description}\n\n#{text}"} + end + end + + defp title(version) do + cond do + not Validations.blank?(version.title) -> version.title + not Validations.blank?(version.meta_title) -> version.meta_title + true -> "none" + end + end + + @doc """ + Reads the published versions of the scope's posts that no vector of the + configured model was stored for, one after the other. Publishes nothing + again. Returns how many the model read. + """ + @spec backfill(Scope.t()) :: non_neg_integer() + def backfill(%Scope{site: %Site{id: site_id}}) do + model = Ollama.embedding_model() + + from(v in PostVersion, + join: p in Post, + on: p.published_version_id == v.id, + where: p.site_id == ^site_id, + where: is_nil(v.embedding_model) or v.embedding_model != ^model, + order_by: p.inserted_at + ) + |> Repo.all() + |> Enum.count(&(embed(&1) == :ok)) + end + + @doc "A vector as stored: little-endian 32-bit floats." + @spec encode([number()]) :: binary() + def encode(vector), do: for(x <- vector, into: <<>>, do: <>) + + @doc "A stored vector as a list of floats." + @spec decode(binary()) :: [float()] + def decode(binary), do: for(<>, do: x) +end diff --git a/lib/feather/content/post_version.ex b/lib/feather/content/post_version.ex index 745b7e5c..92b43df8 100644 --- a/lib/feather/content/post_version.ex +++ b/lib/feather/content/post_version.ex @@ -1,6 +1,9 @@ defmodule Feather.Content.PostVersion do @moduledoc """ A published state of a post, see `Feather.Content.publish/2`. + + `embedding` and `embedding_model` hold what the embedding model read in + it (`Feather.Content.Embedding`); they are not copied from the post. """ use Feather.Schema @@ -34,6 +37,8 @@ defmodule Feather.Content.PostVersion do field :description, :string field :description_generated, :boolean, default: false field :meta_title, :string + field :embedding, :binary + field :embedding_model, :string belongs_to :post, Feather.Content.Post belongs_to :published_by, Feather.Accounts.User diff --git a/lib/feather/content/saving.ex b/lib/feather/content/saving.ex index 16797f41..f3f5f1ee 100644 --- a/lib/feather/content/saving.ex +++ b/lib/feather/content/saving.ex @@ -40,7 +40,9 @@ defmodule Feather.Content.Saving do where `Feather.Content.Description.needed?/2` says so, and a short post's meta title and slug where `Feather.Content.ShortPost.needed?/2` does, outside the transaction, since that can take up to 90 seconds. - Without an answer it is published as it is. + Without an answer it is published as it is. After a post is published, + the embedding model reads its published version in the background + (`Feather.Content.Embedding.start/1`). """ @spec publish(Scope.t(), record) :: {:ok, record} | {:error, :stale | :slug_taken} when record: Post.t() | Page.t() | Project.t() diff --git a/lib/feather/content/versions.ex b/lib/feather/content/versions.ex index e8eaa785..c7e59965 100644 --- a/lib/feather/content/versions.ex +++ b/lib/feather/content/versions.ex @@ -12,7 +12,8 @@ defmodule Feather.Content.Versions do import Ecto.Query, warn: false alias Feather.Accounts.Scope - alias Feather.Content.{Page, PageVersion, Post, PostVersion, Project, ProjectVersion, Slug} + alias Feather.Content.{Embedding, Page, PageVersion, Post, PostVersion, Project, ProjectVersion} + alias Feather.Content.Slug alias Feather.Redirects alias Feather.Repo alias Feather.Sites @@ -44,9 +45,12 @@ defmodule Feather.Content.Versions do def lock_opts, do: @lock_opts @doc false - # publish_stored/3 in its own transaction. - def publish_committed(scope, record, opts \\ []), - do: Repo.transact(fn -> publish_stored(scope, record, opts) end) + # publish_stored/3 in its own transaction; after the commit the model reads a post's new version. + def publish_committed(scope, record, opts \\ []) do + result = Repo.transact(fn -> publish_stored(scope, record, opts) end) + with {:ok, %Post{} = post} <- result, do: Embedding.start(post) + result + end @doc false # Publishes the stored record in the caller's transaction. `state: :unpublished` (the diff --git a/lib/feather/ollama.ex b/lib/feather/ollama.ex index e7297373..d5d0d95b 100644 --- a/lib/feather/ollama.ex +++ b/lib/feather/ollama.ex @@ -7,7 +7,10 @@ defmodule Feather.Ollama do The URL comes from `config :feather, :ollama_url` (`OLLAMA_URL`); without one nothing is asked and every call returns `{:error, :not_configured}`. The model comes from `config :feather, :ollama_model` (`OLLAMA_MODEL`), - by default Gemma 4 26B, which must read images. + by default Gemma 4 26B, which must read images. Embeddings + (`embedding/1`, `POST /api/embed`) come from `config :feather, + :ollama_embedding_model` (`OLLAMA_EMBEDDING_MODEL`), by default + EmbeddingGemma 2. Tests stub the requests with `Req.Test` (`config :feather, :ollama_req_options, plug: {Req.Test, Feather.Ollama}`). """ @@ -17,6 +20,7 @@ defmodule Feather.Ollama do require Logger @default_model "gemma4:26b-a4b-it-qat" + @default_embedding_model "embeddinggemma-2:270m" @timeout 90_000 @max_alt_length 250 @max_description_length 300 @@ -171,6 +175,42 @@ defmodule Feather.Ollama do end end + @doc """ + The embedding of a text by the embedding model: a vector of numbers, + L2-normalised. Ollama cuts a text longer than the model's window. + """ + @spec embedding(String.t()) :: {:ok, [number()]} | {:error, term()} + def embedding(text) when is_binary(text) do + if configured?() do + req() + |> Req.post( + url: "/api/embed", + json: %{model: embedding_model(), input: [text], truncate: true} + ) + |> decode_embedding() + else + {:error, :not_configured} + end + end + + @doc "The name of the embedding model `embedding/1` asks." + @spec embedding_model() :: String.t() + def embedding_model, do: setting(:ollama_embedding_model, @default_embedding_model) + + defp decode_embedding( + {:ok, %Req.Response{status: 200, body: %{"embeddings" => [vector]} = body}} + ) do + if vector?(vector), do: {:ok, vector}, else: log_failure({:invalid_answer, body}) + end + + defp decode_embedding({:ok, %Req.Response{status: 200} = response}), + do: log_failure({:invalid_answer, response.body}) + + defp decode_embedding({:ok, %Req.Response{status: status}}), do: log_failure({:status, status}) + defp decode_embedding({:error, exception}), do: log_failure(exception) + + defp vector?(vector), do: is_list(vector) and vector != [] and Enum.all?(vector, &is_number/1) + defp trimmed(text, max_length), do: text |> String.trim() |> String.slice(0, max_length) defp decode_answer( diff --git a/lib/feather/release.ex b/lib/feather/release.ex index e858f6b8..a177761d 100644 --- a/lib/feather/release.ex +++ b/lib/feather/release.ex @@ -5,6 +5,7 @@ defmodule Feather.Release do bin/feather eval "Feather.Release.migrate()" bin/feather eval 'Feather.Release.create_user("me@example.com", true)' bin/feather eval "Feather.Release.backfill_suggestions()" + bin/feather eval "Feather.Release.backfill_embeddings()" `bin/migrate` and `bin/server` (in `rel/overlays/bin`) wrap the first. """ @@ -107,8 +108,9 @@ defmodule Feather.Release do existed, site by site: the alt texts of all images (`Feather.Media.backfill_alt_texts/1`), and descriptions and short posts' meta titles and slugs of published records without unpublished - changes, by publishing them again (`Feather.Content.backfill_suggestions/1`). - Needs `OLLAMA_URL`. Deploy the sites by hand afterwards. + changes, by publishing them again (`Feather.Content.backfill_suggestions/1`); + the embedding model reads the posts published again right away. Needs + `OLLAMA_URL`. Deploy the sites by hand afterwards. """ def backfill_suggestions, do: each_site_with_model(&backfill_site/1) @@ -122,21 +124,51 @@ defmodule Feather.Release do ) end + @doc """ + Reads the published posts of every site that no embedding of the + configured model was stored for (`Feather.Content.backfill_embeddings/1`), + after a change of the model and for posts published before embeddings + existed. Publishes nothing again. Needs `OLLAMA_URL`. Deploy the sites + by hand afterwards. + """ + def backfill_embeddings, do: each_site_with_model(&backfill_embeddings_of/1) + + defp backfill_embeddings_of(site) do + read = site |> Scope.for_site() |> Feather.Content.backfill_embeddings() + IO.puts("#{site.domain}: posts read: #{read}") + end + defp each_site_with_model(fun) do with_app(fn -> {:ok, _} = Application.ensure_all_started(:req) - if Feather.Ollama.configured?(), - do: Enum.each(Feather.Repo.all(Site), fun), - else: not_configured() + if Feather.Ollama.configured?(), do: each_site_reading_inline(fun), else: not_configured() end) end + defp each_site_reading_inline(fun), + do: reading_inline(fn -> Enum.each(Feather.Repo.all(Site), fun) end) + defp not_configured do IO.puts(:stderr, "OLLAMA_URL is not set.") {:error, :not_configured} end + # eval starts no task supervisor and would end before background readings. + defp reading_inline(fun) do + previous = Application.fetch_env(:feather, :post_embeddings) + Application.put_env(:feather, :post_embeddings, :inline) + + try do + fun.() + after + case previous do + {:ok, mode} -> Application.put_env(:feather, :post_embeddings, mode) + :error -> Application.delete_env(:feather, :post_embeddings) + end + end + end + defp with_app(fun) do load_app() {:ok, _} = Application.ensure_all_started(:ecto_sql) diff --git a/priv/repo/migrations/20261107113721_add_embedding_to_post_versions.exs b/priv/repo/migrations/20261107113721_add_embedding_to_post_versions.exs new file mode 100644 index 00000000..701fb934 --- /dev/null +++ b/priv/repo/migrations/20261107113721_add_embedding_to_post_versions.exs @@ -0,0 +1,10 @@ +defmodule Feather.Repo.Migrations.AddEmbeddingToPostVersions do + use Ecto.Migration + + def change do + alter table(:post_versions) do + add :embedding, :binary + add :embedding_model, :string + end + end +end diff --git a/test/feather/content/embedding_test.exs b/test/feather/content/embedding_test.exs new file mode 100644 index 00000000..5b9c9ad7 --- /dev/null +++ b/test/feather/content/embedding_test.exs @@ -0,0 +1,343 @@ +defmodule Feather.Content.EmbeddingTest do + use Feather.DataCase + + import ExUnit.CaptureIO + + alias Feather.Content + alias Feather.Content.{Embedding, PostVersion} + alias Feather.Release + + @moduletag :capture_log + + setup do + previous = Keyword.take(Application.get_all_env(:feather), [:ollama_url, :post_embeddings]) + Application.put_env(:feather, :ollama_url, "http://ollama.test:11434") + Application.put_env(:feather, :post_embeddings, :inline) + + on_exit(fn -> + Application.delete_env(:feather, :ollama_url) + Application.delete_env(:feather, :post_embeddings) + Enum.each(previous, fn {key, value} -> Application.put_env(:feather, key, value) end) + end) + + %{scope: site_scope_fixture()} + end + + defp stub_embedding(vector) do + test = self() + + Req.Test.stub(Feather.Ollama, fn conn -> + case conn.request_path do + "/api/embed" -> + {:ok, body, conn} = Plug.Conn.read_body(conn) + send(test, {:embed, Jason.decode!(body)}) + Req.Test.json(conn, %{"embeddings" => [vector]}) + + "/api/chat" -> + Plug.Conn.send_resp(conn, 503, "") + end + end) + end + + defp published_version(post), + do: Repo.get!(PostVersion, Repo.reload!(post).published_version_id) + + defp decoded(%PostVersion{embedding: embedding}), do: Embedding.decode(embedding) + + defp assert_vector(actual, expected) do + assert length(actual) == length(expected) + Enum.zip_with(actual, expected, &assert_in_delta(&1, &2, 1.0e-6)) + end + + describe "publishing a post" do + test "asks the model for an embedding of the clustering input and stores it on the new version", + %{scope: scope} do + stub_embedding([0.6, 0.8, 0.0]) + + post = + post_fixture(scope, %{ + title: "Bikes", + description: "Why I ride.", + content: [paragraph("Every day."), paragraph("Rain or shine.")] + }) + + assert_received {:embed, request} + + assert request == %{ + "model" => "embeddinggemma-2:270m", + "input" => [ + "task: clustering | query: Bikes. Why I ride.\n\nEvery day.\n\nRain or shine." + ], + "truncate" => true + } + + version = published_version(post) + assert version.embedding_model == "embeddinggemma-2:270m" + assert_vector(decoded(version), [0.6, 0.8, 0.0]) + end + + test "asks the configured embedding model", %{scope: scope} do + Application.put_env(:feather, :ollama_embedding_model, "nomic-embed-text") + on_exit(fn -> Application.delete_env(:feather, :ollama_embedding_model) end) + stub_embedding([1.0, 0.0]) + + post = post_fixture(scope) + + assert_received {:embed, %{"model" => "nomic-embed-text"}} + assert published_version(post).embedding_model == "nomic-embed-text" + end + + test "reads at most 4000 characters of the text", %{scope: scope} do + stub_embedding([1.0, 0.0]) + post_fixture(scope, %{title: "Long", content: [paragraph(String.duplicate("word ", 2000))]}) + + assert_received {:embed, %{"input" => [input]}} + ["task: clustering | query: Long. ", text] = String.split(input, "\n\n", parts: 2) + assert String.length(text) in 3_990..4_000 + end + + test "a short post uses its meta title, one without either says none", %{scope: scope} do + stub_embedding([1.0, 0.0]) + + post_fixture(scope, %{ + title: nil, + meta_title: "Solar", + slug: "/solar", + content: [paragraph("Sun.")] + }) + + assert_received {:embed, %{"input" => ["task: clustering | query: Solar. \n\nSun."]}} + + post_fixture(scope, %{title: nil, slug: "/rain", content: [paragraph("Rain.")]}) + assert_received {:embed, %{"input" => ["task: clustering | query: none. \n\nRain."]}} + end + + test "embeds every new version, but not a version already read", %{scope: scope} do + stub_embedding([1.0, 0.0]) + post = post_fixture(scope, %{title: "Bikes"}) + assert_received {:embed, _request} + + {:ok, post} = Content.publish(scope, post) + refute_received {:embed, _request} + + {:ok, post} = Content.update_post(scope, post, %{title: "Bicycles"}) + stub_embedding([0.0, 1.0]) + {:ok, _published} = Content.publish(scope, post) + + assert_received {:embed, %{"input" => ["task: clustering | query: Bicycles. " <> _text]}} + assert_vector(decoded(published_version(post)), [0.0, 1.0]) + end + + test "succeeds without an embedding when Ollama is not configured", %{scope: scope} do + Application.delete_env(:feather, :ollama_url) + + version = scope |> post_fixture() |> published_version() + + assert version.embedding == nil + assert version.embedding_model == nil + end + + test "succeeds without an embedding when the model fails", %{scope: scope} do + Req.Test.stub(Feather.Ollama, &Req.Test.transport_error(&1, :econnrefused)) + + version = scope |> post_fixture() |> published_version() + + assert version.embedding == nil + end + + test "reads in the background under the task supervisor by default", %{scope: scope} do + Application.put_env(:feather, :post_embeddings, :background) + test = self() + + Req.Test.stub(Feather.Ollama, fn conn -> + send(test, {:asked, self()}) + receive do: (:answer -> Req.Test.json(conn, %{"embeddings" => [[1.0, 0.0]]})) + end) + + post = post_fixture(scope, %{description: "Set."}) + + assert_receive {:asked, task} + assert task in Task.Supervisor.children(Feather.TaskSupervisor) + ref = Process.monitor(task) + send(task, :answer) + assert_receive {:DOWN, ^ref, :process, ^task, :normal} + + assert_vector(decoded(published_version(post)), [1.0, 0.0]) + end + + test "a post republished by the suggestions backfill is read too", %{scope: scope} do + Application.delete_env(:feather, :ollama_url) + post = post_fixture(scope, %{title: "Bikes"}) + Application.put_env(:feather, :ollama_url, "http://ollama.test:11434") + + Req.Test.stub(Feather.Ollama, fn conn -> + case conn.request_path do + "/api/embed" -> + Req.Test.json(conn, %{"embeddings" => [[1.0, 0.0]]}) + + "/api/chat" -> + Req.Test.json(conn, %{"message" => %{"content" => ~s({"description": "D."})}}) + end + end) + + assert [_republished] = Content.backfill_suggestions(scope) + + version = published_version(post) + assert version.number == 2 + assert_vector(decoded(version), [1.0, 0.0]) + end + + test "a post without plain text gets no embedding", %{scope: scope} do + stub_embedding([1.0, 0.0]) + + version = + scope |> post_fixture(%{title: "Only a title", content: []}) |> published_version() + + refute_received {:embed, _request} + assert version.embedding == nil + end + + test "a page gets no embedding", %{scope: scope} do + stub_embedding([1.0, 0.0]) + page_fixture(scope) + + refute_received {:embed, _request} + end + end + + describe "storing a reading" do + alias Feather.Publishing + + setup %{scope: scope} do + Application.delete_env(:feather, :ollama_url) + %{target: deployment_target_fixture(scope)} + end + + defp deploy_after(target, post) do + :ok = Publishing.record_deploy(target, published_version(post).published_at) + end + + test "of a post published since the last deploy adds no change of its own", + %{scope: scope, target: target} do + :ok = Publishing.record_deploy(target, DateTime.utc_now()) + post = post_fixture(scope) + assert Publishing.undeployed_count(scope) == 1 + + put_embedding(post, [1.0, 0.0]) + + assert Publishing.undeployed_count(scope) == 1 + end + + test "of a post already deployed counts the site as changed", %{scope: scope, target: target} do + post = post_fixture(scope) + deploy_after(target, post) + assert Publishing.undeployed_count(scope) == 0 + + put_embedding(post, [1.0, 0.0]) + + assert Publishing.undeployed_count(scope) == 1 + end + + test "of a scheduled post or a version no longer published changes nothing", + %{scope: scope, target: target} do + scheduled = post_fixture(scope, %{publish_at: DateTime.add(DateTime.utc_now(), 1, :day)}) + post = post_fixture(scope, %{title: "Bikes"}) + first = published_version(post) + {:ok, post} = Content.update_post(scope, post, %{title: "Bicycles"}) + {:ok, post} = Content.publish(scope, post) + deploy_after(target, post) + assert Publishing.undeployed_count(scope) == 0 + + put_embedding(scheduled, [1.0, 0.0]) + :ok = Embedding.store(first, [1.0, 0.0], "embeddinggemma-2:270m") + + assert Publishing.undeployed_count(scope) == 0 + end + end + + describe "backfill_embeddings/1" do + setup %{scope: scope} do + Application.delete_env(:feather, :ollama_url) + unread = post_fixture(scope, %{title: "Unread"}) + + other_model = + scope |> post_fixture(%{title: "Other model"}) |> put_embedding([0.0, 1.0], "old-model") + + read = scope |> post_fixture(%{title: "Read"}) |> put_embedding([0.0, 1.0]) + draft = post_fixture(scope, %{title: "Draft", draft: true}) + Application.put_env(:feather, :ollama_url, "http://ollama.test:11434") + + %{unread: unread, other_model: other_model, read: read, draft: draft} + end + + test "embeds the published versions without a vector of the configured model and leaves the others alone", + %{scope: scope} = c do + stub_embedding([1.0, 0.0]) + + assert Content.backfill_embeddings(scope) == 2 + + for post <- [c.unread, c.other_model] do + version = published_version(post) + assert version.embedding_model == "embeddinggemma-2:270m" + assert_vector(decoded(version), [1.0, 0.0]) + end + + assert_vector(decoded(published_version(c.read)), [0.0, 1.0]) + assert Repo.aggregate(PostVersion, :count) == 3 + end + + test "publishes nothing again", %{scope: scope} = c do + stub_embedding([1.0, 0.0]) + before = published_version(c.unread) + + Content.backfill_embeddings(scope) + + after_backfill = published_version(c.unread) + assert after_backfill.id == before.id + assert after_backfill.number == 1 + assert Repo.reload!(c.unread).updated_at == c.unread.updated_at + end + + test "reads only the scope's site", %{scope: scope} do + Application.delete_env(:feather, :ollama_url) + elsewhere = post_fixture(site_scope_fixture(), %{title: "Elsewhere"}) + Application.put_env(:feather, :ollama_url, "http://ollama.test:11434") + stub_embedding([1.0, 0.0]) + + assert Content.backfill_embeddings(scope) == 2 + assert published_version(elsewhere).embedding == nil + end + + test "leaves out posts without plain text", %{scope: scope} do + Application.delete_env(:feather, :ollama_url) + empty = post_fixture(scope, %{title: "Empty", content: []}) + Application.put_env(:feather, :ollama_url, "http://ollama.test:11434") + stub_embedding([1.0, 0.0]) + + assert Content.backfill_embeddings(scope) == 2 + assert published_version(empty).embedding == nil + end + + test "counts only the versions the model read", %{scope: scope} do + Req.Test.stub(Feather.Ollama, &Req.Test.transport_error(&1, :econnrefused)) + + assert Content.backfill_embeddings(scope) == 0 + end + + test "the release task embeds every site's posts", %{scope: scope} do + stub_embedding([1.0, 0.0]) + + output = capture_io(fn -> assert Release.backfill_embeddings() == :ok end) + + assert output =~ "#{scope.site.domain}: posts read: 2" + end + + test "the release task refuses without a model" do + Application.delete_env(:feather, :ollama_url) + + assert capture_io(:stderr, fn -> + assert Release.backfill_embeddings() == {:error, :not_configured} + end) =~ "OLLAMA_URL is not set." + end + end +end diff --git a/test/feather/ollama_test.exs b/test/feather/ollama_test.exs index 88e4255f..8f7c9fe3 100644 --- a/test/feather/ollama_test.exs +++ b/test/feather/ollama_test.exs @@ -4,12 +4,19 @@ defmodule Feather.OllamaTest do alias Feather.Ollama setup do - previous = Keyword.take(Application.get_all_env(:feather), [:ollama_url, :ollama_model]) + previous = + Keyword.take(Application.get_all_env(:feather), [ + :ollama_url, + :ollama_model, + :ollama_embedding_model + ]) + Application.put_env(:feather, :ollama_url, "http://ollama.test:11434") on_exit(fn -> Application.delete_env(:feather, :ollama_url) Application.delete_env(:feather, :ollama_model) + Application.delete_env(:feather, :ollama_embedding_model) Enum.each(previous, fn {key, value} -> Application.put_env(:feather, key, value) end) end) end @@ -48,4 +55,65 @@ defmodule Feather.OllamaTest do assert {:ok, _alt} = Ollama.alt_text("jpeg", "en") assert_received {:request, %{"model" => "gemma4:26b-a4b-it-qat"}} end + + describe "embedding/1" do + defp stub_embedding(test, response) do + Req.Test.stub(Ollama, fn conn -> + {:ok, body, conn} = Plug.Conn.read_body(conn) + send(test, {:request, conn.method, conn.request_path, Jason.decode!(body)}) + Req.Test.json(conn, response) + end) + end + + test "posts the text to the embed API of EmbeddingGemma 2, truncated to the model's window" do + stub_embedding(self(), %{"embeddings" => [[0.6, 0.8]]}) + + assert Ollama.embedding("task: clustering | query: Bikes.") == {:ok, [0.6, 0.8]} + + assert_received {:request, "POST", "/api/embed", request} + + assert request == %{ + "model" => "embeddinggemma-2:270m", + "input" => ["task: clustering | query: Bikes."], + "truncate" => true + } + end + + test "asks the configured embedding model, an empty setting counts as none" do + Application.put_env(:feather, :ollama_embedding_model, "nomic-embed-text") + stub_embedding(self(), %{"embeddings" => [[1.0]]}) + assert {:ok, _vector} = Ollama.embedding("text") + assert_received {:request, _method, _path, %{"model" => "nomic-embed-text"}} + + Application.put_env(:feather, :ollama_embedding_model, "") + assert {:ok, _vector} = Ollama.embedding("text") + assert_received {:request, _method, _path, %{"model" => "embeddinggemma-2:270m"}} + end + + @tag :capture_log + test "an answer without a vector of numbers is an error" do + for response <- [ + %{"embeddings" => []}, + %{"embeddings" => [[]]}, + %{"embeddings" => [["a"]]}, + %{"error" => "no model"} + ] do + stub_embedding(self(), response) + assert {:error, _reason} = Ollama.embedding("text") + end + end + + @tag :capture_log + test "a failing model is an error with its status" do + Req.Test.stub(Ollama, &Plug.Conn.send_resp(&1, 503, "")) + + assert Ollama.embedding("text") == {:error, {:status, 503}} + end + + test "without a URL nothing is asked" do + Application.delete_env(:feather, :ollama_url) + + assert Ollama.embedding("text") == {:error, :not_configured} + end + end end diff --git a/test/feather/release_test.exs b/test/feather/release_test.exs index f20a4f8d..f5f605da 100644 --- a/test/feather/release_test.exs +++ b/test/feather/release_test.exs @@ -3,6 +3,7 @@ defmodule Feather.ReleaseTest do import ExUnit.CaptureIO + alias Feather.Content.PostVersion alias Feather.Release describe "set_target_config/2" do @@ -34,10 +35,35 @@ defmodule Feather.ReleaseTest do describe "backfill_suggestions/0" do setup do - previous = Application.get_env(:feather, :ollama_url) - on_exit(fn -> Application.put_env(:feather, :ollama_url, previous) end) + previous = Keyword.take(Application.get_all_env(:feather), [:ollama_url, :post_embeddings]) + + on_exit(fn -> + Application.delete_env(:feather, :ollama_url) + Application.delete_env(:feather, :post_embeddings) + Enum.each(previous, fn {key, value} -> Application.put_env(:feather, key, value) end) + end) + end + + defp stub_model(test) do + Req.Test.stub(Feather.Ollama, fn conn -> + {:ok, body, conn} = Plug.Conn.read_body(conn) + + case conn.request_path do + "/api/embed" -> + send(test, {:embedded_in, self()}) + Req.Test.json(conn, %{"embeddings" => [[1.0, 0.0]]}) + + "/api/chat" -> + answer = chat_answer(Jason.decode!(body)["format"]["required"]) + Req.Test.json(conn, %{"message" => %{"content" => Jason.encode!(answer)}}) + end + end) end + defp chat_answer(["alt"]), do: %{"alt" => "Grau"} + defp chat_answer(["description"]), do: %{"description" => "Solar."} + defp chat_answer(["title", "slug"]), do: %{"title" => "Solar", "slug" => "solar"} + @tag :capture_log test "fills in alt texts and republishes short posts of every site" do Application.delete_env(:feather, :ollama_url) @@ -47,18 +73,7 @@ defmodule Feather.ReleaseTest do Application.put_env(:feather, :ollama_url, "http://ollama.test:11434") - Req.Test.stub(Feather.Ollama, fn conn -> - {:ok, body, conn} = Plug.Conn.read_body(conn) - - answer = - case Jason.decode!(body)["format"]["required"] do - ["alt"] -> %{"alt" => "Grau"} - ["description"] -> %{"description" => "Solar."} - ["title", "slug"] -> %{"title" => "Solar", "slug" => "solar"} - end - - Req.Test.json(conn, %{"message" => %{"content" => Jason.encode!(answer)}}) - end) + stub_model(self()) output = capture_io(fn -> assert Release.backfill_suggestions() == :ok end) @@ -67,6 +82,24 @@ defmodule Feather.ReleaseTest do assert Repo.reload!(post).slug == "/solar" end + @tag :capture_log + test "reads the republished posts before it returns, as a release runs no background tasks" do + Application.delete_env(:feather, :ollama_url) + scope = site_scope_fixture() + post = post_fixture(scope, %{title: nil, slug: nil, content: [paragraph("Solar.")]}) + Application.put_env(:feather, :ollama_url, "http://ollama.test:11434") + Application.put_env(:feather, :post_embeddings, :background) + stub_model(self()) + + capture_io(fn -> assert Release.backfill_suggestions() == :ok end) + + assert_received {:embedded_in, pid} + assert pid == self() + version = Repo.get!(PostVersion, Repo.reload!(post).published_version_id) + assert version.embedding_model == "embeddinggemma-2:270m" + assert Application.get_env(:feather, :post_embeddings) == :background + end + test "refuses without a model" do Application.delete_env(:feather, :ollama_url) diff --git a/test/support/fixtures/content_fixtures.ex b/test/support/fixtures/content_fixtures.ex index 45bab4f6..7d873c03 100644 --- a/test/support/fixtures/content_fixtures.ex +++ b/test/support/fixtures/content_fixtures.ex @@ -5,7 +5,8 @@ defmodule Feather.ContentFixtures do The records are published (version 1) unless `draft: true` is given. """ - alias Feather.Content + alias Feather.{Content, Ollama, Repo} + alias Feather.Content.{Embedding, PostVersion} def unique_slug(prefix \\ "item"), do: "/#{prefix}-#{System.unique_integer([:positive])}" @@ -60,6 +61,22 @@ defmodule Feather.ContentFixtures do publish_unless_draft(scope, project, draft) end + @doc """ + Stores `vector`, scaled to length 1 as Ollama returns it, as the + embedding of the post's published version, by default of the configured + model. + """ + def put_embedding(post, vector, model \\ Ollama.embedding_model()) do + length = :math.sqrt(Enum.reduce(vector, 0.0, &(&1 * &1 + &2))) + + :ok = + PostVersion + |> Repo.get!(post.published_version_id) + |> Embedding.store(Enum.map(vector, &(&1 / length)), model) + + post + end + defp pop_draft(attrs), do: attrs |> Map.new() |> Map.pop(:draft, false) defp publish_unless_draft(_scope, record, true), do: record From fe5cac1a7c19080856265f2bf23670f45bfc8250 Mon Sep 17 00:00:00 2001 From: Robert Curth Date: Sat, 7 Nov 2026 15:51:31 +0100 Subject: [PATCH 3/4] List related posts first in the post navigation RelatedPosts ranks a site's posts by the cosine similarity of their published versions' embeddings plus a small bonus for rare shared tags, ties going to the post closer in date. The export ranks every pair once before its render jobs start, the preview only the post it shows. Both themes list up to three related posts, with their publish year, above the newer and older post, which never count among them, and every link in the post navigation now names a short post by its meta title, with the emoji in a column of its own. Closes #522 Co-Authored-By: Claude Opus 5.5 --- lib/feather/content/related_posts.ex | 113 +++++++ lib/feather/static_site/export.ex | 7 +- lib/feather/static_site/renderer.ex | 1 + lib/feather/static_site/site_data.ex | 53 +++- lib/feather/static_site/theme.ex | 7 +- lib/feather/static_site/theme_helpers.ex | 30 +- .../themes/feather_page/post.html.eex | 38 ++- .../themes/simple_sidebar/post.html.eex | 14 +- .../gettext/de/LC_MESSAGES/default.po | 24 +- priv/static_site/gettext/default.pot | 24 +- .../static_site/themes/feather_page/theme.css | 77 ++++- .../themes/simple_sidebar/theme.css | 44 ++- test/feather/content/related_posts_test.exs | 166 +++++++++++ .../static_site/export_related_posts_test.exs | 276 ++++++++++++++++++ .../static_site/theme_helpers_test.exs | 18 ++ 15 files changed, 837 insertions(+), 55 deletions(-) create mode 100644 lib/feather/content/related_posts.ex create mode 100644 test/feather/content/related_posts_test.exs create mode 100644 test/feather/static_site/export_related_posts_test.exs diff --git a/lib/feather/content/related_posts.ex b/lib/feather/content/related_posts.ex new file mode 100644 index 00000000..21769823 --- /dev/null +++ b/lib/feather/content/related_posts.ex @@ -0,0 +1,113 @@ +defmodule Feather.Content.RelatedPosts do + @moduledoc """ + Ranks a site's posts as related posts of each other (ADR 0036): the + other posts closest in content by the cosine similarity of their published + versions' embeddings (`Feather.Content.Embedding`), plus 0.06 per + shared tag, at most 0.12. A tag counts only while at most three posts + or at most a fifth of the due posts handed in carry it, whichever is more, + so tags on many posts add nothing. On equal scores the post closer in + publish date wins. Posts without a vector of the model take no part on + either side. + + Everything comes from the published versions (preloaded): their + vectors, tags and publish dates. The preview ranks the same published + versions, but among the posts it shows, so drafts and changed publish + dates can change its list. `all/3` compares every pair once + (the export), `of/4` one post with all others (the preview); both keep + the `limit` best. + """ + + alias Feather.Content.{Embedding, Post, PostVersion, Tags} + + @tag_bonus 0.06 + @max_tag_bonus 0.12 + @min_tag_limit 3 + + @doc "The ids of the `limit` best related posts of each post with a vector, best first, by post id." + @spec all([Post.t()], String.t(), pos_integer()) :: %{Ecto.UUID.t() => [Ecto.UUID.t()]} + def all(posts, model, limit) do + readings = readings(posts, model) + scored = readings |> Map.values() |> pair_scores(%{}) + + Map.new(readings, fn {id, reading} -> + {id, best(reading, Map.get(scored, id, []), limit)} + end) + end + + @doc "The ids of the `limit` best related posts of one of the posts, best first; none without a vector." + @spec of([Post.t()], Post.t(), String.t(), pos_integer()) :: [Ecto.UUID.t()] + def of(posts, %Post{id: id}, model, limit) do + case Map.pop(readings(posts, model), id) do + {nil, _readings} -> + [] + + {reading, others} -> + best(reading, Enum.map(Map.values(others), &{&1, score(reading, &1)}), limit) + end + end + + defp readings(posts, model) do + versions = for %Post{published_version: %PostVersion{} = v} = post <- posts, do: {post.id, v} + counting = counting_tags(versions) + + for {id, %PostVersion{embedding: embedding, embedding_model: ^model} = version} <- versions, + is_binary(embedding), + into: %{} do + {id, + %{ + id: id, + vector: Embedding.decode(embedding), + tags: version |> Tags.tag_list() |> MapSet.new() |> MapSet.intersection(counting), + publish_at: version.publish_at + }} + end + end + + defp counting_tags(versions) do + limit = max(@min_tag_limit, length(versions) / 5) + + for {tag, count} <- + versions |> Enum.flat_map(fn {_id, v} -> Tags.tag_list(v) end) |> Enum.frequencies(), + count <= limit, + into: MapSet.new(), + do: tag + end + + defp pair_scores([], scored), do: scored + + defp pair_scores([reading | rest], scored) do + scored = + Enum.reduce(rest, scored, fn other, scored -> + score = score(reading, other) + + scored + |> Map.update(reading.id, [{other, score}], &[{other, score} | &1]) + |> Map.update(other.id, [{reading, score}], &[{reading, score} | &1]) + end) + + pair_scores(rest, scored) + end + + defp best(reading, scored_others, limit) do + scored_others + |> Enum.sort_by(fn {other, score} -> {-score, date_distance(reading, other), other.id} end) + |> Enum.take(limit) + |> Enum.map(fn {other, _score} -> other.id end) + end + + defp score(reading, other), + do: dot(reading.vector, other.vector, 0.0) + tag_bonus(reading.tags, other.tags) + + defp dot([x | xs], [y | ys], sum), do: dot(xs, ys, sum + x * y) + defp dot(_rest, _other_rest, sum), do: sum + + defp tag_bonus(tags, other_tags) do + shared = tags |> MapSet.intersection(other_tags) |> MapSet.size() + min(@tag_bonus * shared, @max_tag_bonus) + end + + defp date_distance(%{publish_at: %DateTime{} = a}, %{publish_at: %DateTime{} = b}), + do: abs(DateTime.diff(a, b)) + + defp date_distance(_reading, _other), do: :undated +end diff --git a/lib/feather/static_site/export.ex b/lib/feather/static_site/export.ex index 162698b8..1c5128a0 100644 --- a/lib/feather/static_site/export.ex +++ b/lib/feather/static_site/export.ex @@ -51,7 +51,12 @@ defmodule Feather.StaticSite.Export do """ @spec run(Site.t(), Routes.t(), Sink.t(), keyword()) :: :ok def run(%Site{} = site, %Routes{} = routes, sink, opts \\ []) do - data = site |> SiteData.load(Keyword.take(opts, [:now])) |> SiteData.prerender() + data = + site + |> SiteData.load(Keyword.take(opts, [:now])) + |> SiteData.prerender() + |> SiteData.put_related_posts() + redirects = SiteData.redirects(data) documents = documents(data) diff --git a/lib/feather/static_site/renderer.ex b/lib/feather/static_site/renderer.ex index 1f22b4d8..656dbe50 100644 --- a/lib/feather/static_site/renderer.ex +++ b/lib/feather/static_site/renderer.ex @@ -117,6 +117,7 @@ defmodule Feather.StaticSite.Renderer do header_image: post.header_image, newer_post: newer, older_post: older, + related_posts: SiteData.related_posts(data, post), replies: Map.get(data.replies, post.id, []), reply_handle: Fediverse.handle(data.site) }) diff --git a/lib/feather/static_site/site_data.ex b/lib/feather/static_site/site_data.ex index e1c0757d..a6b5f3ad 100644 --- a/lib/feather/static_site/site_data.ex +++ b/lib/feather/static_site/site_data.ex @@ -10,12 +10,13 @@ defmodule Feather.StaticSite.SiteData do import Ecto.Query, warn: false alias Feather.Accounts.Scope - alias Feather.{Books, Content, Diagrams, Fediverse, Highlighting, Media, Redirects, Sites} + alias Feather.{Books, Content, Diagrams, Fediverse, Highlighting, Media, Ollama, Redirects} alias Feather.Books.Book - alias Feather.Content.{Blocks, Page, Post, Project} + alias Feather.Content.{Blocks, Page, Post, Project, RelatedPosts} alias Feather.Media.Image alias Feather.Redirects.Redirect alias Feather.Repo + alias Feather.Sites alias Feather.Sites.{NavigationItem, Site, SocialMediaLink} alias Feather.StaticSite.Themes @@ -26,6 +27,7 @@ defmodule Feather.StaticSite.SiteData do follower_count: 0, following_count: 0, posts: [], + related: nil, replies: %{}, member_replies: [], pages: [], @@ -46,6 +48,7 @@ defmodule Feather.StaticSite.SiteData do follower_count: non_neg_integer(), following_count: non_neg_integer(), posts: [Post.t()], + related: %{Ecto.UUID.t() => [Ecto.UUID.t()]} | nil, replies: %{Ecto.UUID.t() => [Feather.Fediverse.Mention.t()]}, member_replies: [Feather.Fediverse.MemberReply.t()], pages: [Page.t()], @@ -64,10 +67,15 @@ defmodule Feather.StaticSite.SiteData do @project_preloads [:header_image, :thumbnail_image] @book_preloads [:cover_image] + @related_count 3 + # Enough to still show three after leaving out the newer and the older post. + @ranked_count @related_count + 2 + @doc """ Loads the data of a site. `posts` are the posts whose `publish_at` has - passed, newest first; `pages` are all pages, the homepage included; a - book's `post` is its review among `posts`, nil if it is none of them; the + passed, newest first; `related` is nil until `put_related_posts/1`; + `pages` are all pages, the homepage included; a book's `post` is its + review among `posts`, nil if it is none of them; the navigation leaves out pages that are not among them; `now` is the time the site is exported at; `follower_count` counts the followers of the site's fediverse account, `following_count` the accounts that accepted @@ -242,6 +250,43 @@ defmodule Feather.StaticSite.SiteData do end end + @doc """ + Ranks all posts once (`Feather.Content.RelatedPosts.all/3`) and keeps + the ids of each post's best related posts in `related`, for an export + that renders every post. The vectors themselves stay out of the data, + which every render job gets a copy of. + """ + @spec put_related_posts(t()) :: t() + def put_related_posts(%__MODULE__{posts: posts} = data), + do: %{data | related: RelatedPosts.all(posts, Ollama.embedding_model(), @ranked_count)} + + @doc """ + Up to three related posts of a post among the posts, best first, by the + published versions (`Feather.Content.RelatedPosts`). The preview ranks + the same versions among the posts it shows, so drafts and changed + publish dates can change its list. Its newer and older post (`neighbours/2`), + which the post navigation shows anyway, are left out. Without + `put_related_posts/1` it ranks just this post. None for a draft, a post + without an embedding and posts not in the list. + """ + @spec related_posts(t(), Post.t()) :: [Post.t()] + def related_posts(%__MODULE__{posts: posts} = data, %Post{} = post) do + posts_by_id = Map.new(posts, &{&1.id, &1}) + {newer, older} = neighbours(data, post) + shown = for %Post{id: id} <- [newer, older], do: id + + data + |> ranked_ids(post) + |> Enum.reject(&(&1 in shown)) + |> Enum.take(@related_count) + |> Enum.map(&Map.fetch!(posts_by_id, &1)) + end + + defp ranked_ids(%__MODULE__{related: nil, posts: posts}, post), + do: RelatedPosts.of(posts, post, Ollama.embedding_model(), @ranked_count) + + defp ranked_ids(%__MODULE__{related: related}, %Post{id: id}), do: Map.get(related, id, []) + @doc """ The images the exported site uses, without duplicates: the images embedded in the content of its posts, pages and projects, their header diff --git a/lib/feather/static_site/theme.ex b/lib/feather/static_site/theme.ex index 6de3a0aa..39b6a94c 100644 --- a/lib/feather/static_site/theme.ex +++ b/lib/feather/static_site/theme.ex @@ -49,8 +49,11 @@ defmodule Feather.StaticSite.Theme do and each view its own: * `c:home/1` - `posts`, `current_page`, `total_pages` - * `c:post/1` - `post`, `newer_post`, `older_post`, `replies` (the - approved replies to it, oldest first, see + * `c:post/1` - `post`, `newer_post`, `older_post`, `related_posts` + (`Feather.StaticSite.SiteData.related_posts/2`, listed first in + the post navigation, left out when empty; label every link there + with `ThemeHelpers.post_nav_label/2`), + `replies` (the approved replies to it, oldest first, see `Feather.Fediverse.Mentions`, each with the members' public replies to it in `member_replies`, shown below it and marked as by the author) and `reply_handle` (the site's fediverse handle, nil without diff --git a/lib/feather/static_site/theme_helpers.ex b/lib/feather/static_site/theme_helpers.ex index d27a5a5b..f772ae69 100644 --- a/lib/feather/static_site/theme_helpers.ex +++ b/lib/feather/static_site/theme_helpers.ex @@ -483,6 +483,30 @@ defmodule Feather.StaticSite.ThemeHelpers do {:safe, [mark, escape(text)]} end + @doc """ + A post as the post navigation links it: its emoji in a column of its + own, empty without one, beside its title or a short post's meta title + (`post_meta_title/1`); with `year: true` its publish year after that. + """ + @spec post_nav_label(Content.Post.t(), keyword()) :: Phoenix.HTML.safe() + def post_nav_label(post, opts \\ []) do + year = + if opts[:year], + do: [~s( ), to_string(post.publish_at.year), ""], + else: [] + + {:safe, + [ + ~s(), + escape(post_meta_title(post)), + "", + year, + "" + ]} + end + @description_length 155 @doc """ @@ -527,9 +551,9 @@ defmodule Feather.StaticSite.ThemeHelpers do end @doc """ - What the document title, `og:title`, the Markdown alternates and - `llms.txt` name a post by: a short post's meta title, else its label - (`post_label/1`). + What the document title, `og:title`, the Markdown alternates, + `llms.txt` and the post navigation name a post by: a short post's meta + title, else its label (`post_label/1`). """ @spec post_meta_title(Content.Post.t()) :: String.t() def post_meta_title(post) do diff --git a/lib/feather/static_site/themes/feather_page/post.html.eex b/lib/feather/static_site/themes/feather_page/post.html.eex index 260ce747..767455a6 100644 --- a/lib/feather/static_site/themes/feather_page/post.html.eex +++ b/lib/feather/static_site/themes/feather_page/post.html.eex @@ -14,17 +14,29 @@ <% true -> %> <% end %> - <%= if @newer_post do %> -

- <%= gettext("Newer post") %> - <%= emoji_label(@newer_post.emoji, post_label(@newer_post)) %> -

- <% end %> - <%= if @older_post do %> -

- <%= gettext("Older post") %> - <%= emoji_label(@older_post.emoji, post_label(@older_post)) %> -

- <% end %> -

<%= gettext("All posts") %>

+
+ <%= if @related_posts != [] do %> +
+ <%= gettext("Related posts") %> + +
+ <% end %> + <%= if @newer_post do %> +

+ <%= gettext("Newer post") %> + <%= post_nav_label(@newer_post) %> +

+ <% end %> + <%= if @older_post do %> +

+ <%= gettext("Older post") %> + <%= post_nav_label(@older_post) %> +

+ <% end %> +

<%= gettext("All posts") %>

+
diff --git a/lib/feather/static_site/themes/simple_sidebar/post.html.eex b/lib/feather/static_site/themes/simple_sidebar/post.html.eex index bd8f57fc..fb633727 100644 --- a/lib/feather/static_site/themes/simple_sidebar/post.html.eex +++ b/lib/feather/static_site/themes/simple_sidebar/post.html.eex @@ -16,16 +16,26 @@ <%= if @reply_handle do %><%= replies(%{replies: @replies, handle: @reply_handle, routes: @routes, site: @site}) %><% end %>