diff --git a/README.md b/README.md index 620309e..8189c5c 100644 --- a/README.md +++ b/README.md @@ -47,11 +47,16 @@ Experimental retrieval eval over scraped Markdown pages: ```bash just eval-chunks just eval-index qwen +just geocode-pages just eval-generate 10 just eval qwen,sparse just eval qwen4b_rerank,qwen4b_hybrid,qwen4b_hybrid_rerank ``` +`just geocode-pages` geocodes each page once, enabling `qwen_hybrid_geo` — a +chunk retrieval method that nudges ranking by distance to the question's +location (see "Geo-Aware Ranking" in [docs/architecture-decisions.md](docs/architecture-decisions.md)). + ## Shape ```text diff --git a/data/db/pages.db b/data/db/pages.db index 2a502cd..c579e01 100644 Binary files a/data/db/pages.db and b/data/db/pages.db differ diff --git a/docs/architecture-decisions.md b/docs/architecture-decisions.md index 18483a2..28e9b5b 100644 --- a/docs/architecture-decisions.md +++ b/docs/architecture-decisions.md @@ -50,6 +50,36 @@ Measured Qwen3 reranking was harmful and slow in our setup. Keep rerank methods available for experiments, but do not treat rerank as default until model usage, prompt, and input formatting are diagnosed. +## Geo-Aware Ranking + +Pages can carry a `latitude`/`longitude`, geocoded once via `just +geocode-pages` into `page_locations` (`sql/eval.sql`, keyed by `page_id`): a +local LLM names the one real-world place a page describes, and that name is +geocoded through OpenStreetMap Nominatim (`src.shared.geocode`). Reruns leave +existing coordinates alone unless `--force` is passed. + +At query time the same LLM step runs on the question; if it names a place, +every candidate chunk's existing score is rescaled by distance to it. This +wraps the current retriever rather than replacing it: `qwen_hybrid_geo` +(`src.retrieval.retrievers.geo.GeoBoostRetriever`) is `qwen_hybrid` plus a +distance multiplier, registered in `src.retrieval.methods` like any other +method so `src.eval` can score it directly against the rest: + +```text +final_score = score * ((1 - weight) + weight * exp(-distance_km / decay_km)) +``` + +Default `weight=0.25` floors the penalty at 75% of the original score, so +distance can only ever nudge ranking, never override a strong text match — +the same weighted-fusion shape as hybrid vector+sparse scoring (see "Hybrid +History"), not a hard radius filter that could drop the right chunk over one +bad geocode. A chunk or question with no coordinates leaves the score +untouched. + +`qwen_hybrid_geo` only helps once `just geocode-pages` has populated real +coordinates. Nominatim was chosen over the Google Geocoding API for no API +key or billing risk, at the cost of an external network dependency. + ## Chunk Storage SQLite is source of truth for page chunks. Chroma is derived vector cache. @@ -194,7 +224,9 @@ the query: the first version burned its whole budget re-searching one term. ## Local-Only Inference Every model call in the repo runs on local hardware. There is no hosted-model -or credential path in the tree. +or credential path in the tree, except geo-aware ranking's Nominatim lookup +(see "Geo-Aware Ranking"), which is a plain geocoding HTTP call, not a model +call or credential. - Embeddings and reranking run through sentence-transformers; the agentic sufficiency judge, the OKF navigator and generator, the evidence-equivalence diff --git a/justfile b/justfile index b0d70d4..0b6e95b 100644 --- a/justfile +++ b/justfile @@ -34,6 +34,9 @@ eval-phase2 METHODS="phase2-nemotron" OUTPUT="docs/retrieval-results-phase2.md": eval-generate LIMIT="10": uv run python -m src.eval.generate_dataset --limit {{LIMIT}} +geocode-pages *FLAGS: + uv run python -m src.eval.geocode_pages {{FLAGS}} + eval METHODS="qwen": uv run python -m src.eval.evaluate --methods {{METHODS}} diff --git a/sql/eval.sql b/sql/eval.sql index e21bc1e..559c03f 100644 --- a/sql/eval.sql +++ b/sql/eval.sql @@ -28,6 +28,12 @@ create table if not exists eval_relevant_chunks ( primary key (question_id, chunk_id) ); +create table if not exists page_locations ( + page_id text primary key references page_metadata(id) on delete cascade, + latitude real not null, + longitude real not null +); + create index if not exists idx_page_chunks_page_id on page_chunks(page_id); diff --git a/src/db/pages.py b/src/db/pages.py index 73d6022..534fca9 100644 --- a/src/db/pages.py +++ b/src/db/pages.py @@ -1,13 +1,22 @@ from __future__ import annotations import sqlite3 +from dataclasses import dataclass from pathlib import Path from src.shared.env import ROOT, load_local_env, load_yaml +from src.shared.geocode import Coordinates CONFIG = load_yaml(Path(__file__).with_name("config.yaml")) +@dataclass(frozen=True) +class PageForGeocoding: + id: str + title: str | None + markdown: str + + def raw_pages_db_path() -> Path: load_local_env() path = ROOT / CONFIG["raw_pages_db"] @@ -48,6 +57,52 @@ def initialize_page_artifacts_db() -> None: ) +def load_pages_for_geocoding(force: bool = False) -> list[PageForGeocoding]: + sql = """ + select m.id, m.title, c.markdown + from page_metadata m + join page_markdown_content c on c.page_id = m.id + where m.error is null + and m.page_kind != 'empty' + and length(trim(c.markdown)) > 0 + """ + if not force: + sql += " and m.id not in (select page_id from page_locations)" + with connect_pages() as conn: + rows = conn.execute(sql).fetchall() + return [PageForGeocoding(row["id"], row["title"], row["markdown"]) for row in rows] + + +def upsert_page_location(page_id: str, latitude: float, longitude: float) -> None: + with connect_pages() as conn: + conn.execute( + "insert into page_locations (page_id, latitude, longitude) " + "values (?, ?, ?) " + "on conflict(page_id) do update set " + "latitude = excluded.latitude, longitude = excluded.longitude", + (page_id, latitude, longitude), + ) + + +def load_chunk_coordinates(chunk_ids: list[str]) -> dict[str, Coordinates]: + if not chunk_ids: + return {} + placeholders = ", ".join("?" for _ in chunk_ids) + with connect_pages() as conn: + rows = conn.execute( + f""" + select pc.id as chunk_id, pl.latitude, pl.longitude + from page_chunks pc + join page_locations pl on pl.page_id = pc.page_id + where pc.id in ({placeholders}) + """, + chunk_ids, + ).fetchall() + return { + row["chunk_id"]: Coordinates(row["latitude"], row["longitude"]) for row in rows + } + + def _drop_page_chunk_summary(conn: sqlite3.Connection) -> None: try: conn.execute("alter table page_chunks drop column summary") diff --git a/src/eval/config.yaml b/src/eval/config.yaml index 8f30be2..ecc22e2 100644 --- a/src/eval/config.yaml +++ b/src/eval/config.yaml @@ -14,3 +14,5 @@ model_aliases: question_model_num_ctx: 8192 question_model_num_predict: 4096 question_model_reasoning: false +geocode_model: gpt-oss:20b +geocode_max_chars: 4000 diff --git a/src/eval/geocode_pages.py b/src/eval/geocode_pages.py new file mode 100644 index 0000000..a8397eb --- /dev/null +++ b/src/eval/geocode_pages.py @@ -0,0 +1,52 @@ +import argparse +from pathlib import Path + +from src.db.pages import ( + initialize_page_artifacts_db, + load_pages_for_geocoding, + upsert_page_location, +) +from src.shared.env import load_local_env, load_yaml +from src.shared.geocode import NominatimGeocoder, locate_text +from src.shared.llm import LocalOllamaStructuredLlm + + +__all__ = ["geocode_pages"] + +CONFIG = load_yaml(Path(__file__).with_name("config.yaml")) + + +def geocode_pages(*, force: bool = False) -> None: + load_local_env() + initialize_page_artifacts_db() + llm = LocalOllamaStructuredLlm(CONFIG["geocode_model"], method="function_calling") + geocoder = NominatimGeocoder() + + pages = load_pages_for_geocoding(force=force) + max_chars = CONFIG["geocode_max_chars"] + updated = 0 + for page in pages: + text = f"{page.title or ''}\n\n{page.markdown}"[:max_chars] + coords = locate_text(llm, geocoder, text) + if coords is None: + continue + upsert_page_location(page.id, coords.latitude, coords.longitude) + updated += 1 + print(f"{page.id}: {coords.latitude:.5f}, {coords.longitude:.5f}") + + print(f"geocoded {updated} of {len(pages)} pages") + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument( + "--force", + action="store_true", + help="Re-geocode pages that already have a stored location.", + ) + args = parser.parse_args() + geocode_pages(force=args.force) + + +if __name__ == "__main__": + main() diff --git a/src/retrieval/config.yaml b/src/retrieval/config.yaml index 86d6208..91bd23f 100644 --- a/src/retrieval/config.yaml +++ b/src/retrieval/config.yaml @@ -32,3 +32,7 @@ agentic_tools_search_limit: 5 # The richer ToolAction schema is unreliable via tool calls (1/3) but solid via # json_schema (9/10); the plain sufficiency judge is the other way around. agentic_tools_structured_method: json_schema +geo_model: gpt-oss:20b +geo_structured_method: function_calling +geo_weight: 0.25 +geo_decay_km: 50 diff --git a/src/retrieval/methods.py b/src/retrieval/methods.py index 2e22149..353f075 100644 --- a/src/retrieval/methods.py +++ b/src/retrieval/methods.py @@ -4,15 +4,19 @@ from pathlib import Path from typing import Callable +from src.db.pages import load_chunk_coordinates from src.indexing.chunk_text import CONTEXTUAL_CHUNK_VERSION, LEGACY_CHUNK_VERSION from src.retrieval.base import Retriever from src.retrieval.retrievers.agentic import AgenticRetriever, default_judge from src.retrieval.retrievers.agentic_tools import AgenticToolRetriever from src.retrieval.retrievers.fusion import WeightedScoreFusionRetriever +from src.retrieval.retrievers.geo import GeoBoostRetriever from src.retrieval.retrievers.rerank import CrossEncoderRerankRetriever from src.retrieval.retrievers.sparse import SparseRetriever from src.retrieval.retrievers.vector_chunks import VectorChunkRetriever from src.shared.env import load_yaml +from src.shared.geocode import NominatimGeocoder +from src.shared.llm import LocalOllamaStructuredLlm from src.vector_store.chunks import collection_ready, enabled_provider_names CONFIG = load_yaml(Path(__file__).with_name("config.yaml")) @@ -155,6 +159,13 @@ def _provider_specs( provider="qwen", chunk_version=chunk_version, ) + name = f"qwen_hybrid_geo{suffix}" + specs[name] = RetrieverSpec( + name, + lambda version=chunk_version: _hybrid_geo("qwen", version), + provider="qwen", + chunk_version=chunk_version, + ) if provider_name in {"qwen", "nemotron"}: name = f"{provider_name}_hybrid_agentic{suffix}" specs[name] = RetrieverSpec( @@ -220,6 +231,24 @@ def _hybrid( ) +def _hybrid_geo( + provider_name: str, + chunk_version: str = LEGACY_CHUNK_VERSION, +) -> GeoBoostRetriever: + suffix = _version_suffix(chunk_version) + return GeoBoostRetriever( + name=f"{provider_name}_hybrid_geo{suffix}", + base_retriever=_hybrid(provider_name, chunk_version), + llm=LocalOllamaStructuredLlm( + CONFIG["geo_model"], method=CONFIG["geo_structured_method"] + ), + geocoder=NominatimGeocoder(), + chunk_coordinates=load_chunk_coordinates, + weight=CONFIG["geo_weight"], + decay_km=CONFIG["geo_decay_km"], + ) + + def _vector_rerank( provider_name: str, chunk_version: str = LEGACY_CHUNK_VERSION, diff --git a/src/retrieval/retrievers/geo.py b/src/retrieval/retrievers/geo.py new file mode 100644 index 0000000..2b6b03e --- /dev/null +++ b/src/retrieval/retrievers/geo.py @@ -0,0 +1,68 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import Callable + +from src.retrieval.base import RankedChunk, Retriever +from src.shared.geo_boost import distance_multiplier, haversine_km +from src.shared.geocode import Coordinates, GeocodeProvider, locate_text +from src.shared.llm import StructuredLlm + +__all__ = ["GeoBoostRetriever"] + + +@dataclass(frozen=True) +class GeoBoostRetriever: + name: str + base_retriever: Retriever + llm: StructuredLlm + geocoder: GeocodeProvider + chunk_coordinates: Callable[[list[str]], dict[str, Coordinates]] + weight: float + decay_km: float + + def retrieve(self, query: str, limit: int) -> list[RankedChunk]: + candidates = self.base_retriever.retrieve(query, limit) + return self._boost(query, candidates) + + def retrieve_batch( + self, + queries: list[str], + limit: int, + ) -> dict[int, list[RankedChunk]]: + rankings = self.base_retriever.retrieve_batch(queries, limit) + return { + index: self._boost(queries[index], chunks) + for index, chunks in rankings.items() + } + + def _boost(self, query: str, chunks: list[RankedChunk]) -> list[RankedChunk]: + if not chunks: + return chunks + question_coords = locate_text(self.llm, self.geocoder, query) + if question_coords is None: + return chunks + + coords_by_chunk = self.chunk_coordinates([chunk.id for chunk in chunks]) + boosted = [ + RankedChunk( + id=chunk.id, + score=chunk.score + * self._multiplier(chunk.id, coords_by_chunk, question_coords), + text=chunk.text, + ) + for chunk in chunks + ] + return sorted(boosted, key=lambda chunk: chunk.score, reverse=True) + + def _multiplier( + self, + chunk_id: str, + coords_by_chunk: dict[str, Coordinates], + question_coords: Coordinates, + ) -> float: + coords = coords_by_chunk.get(chunk_id) + if coords is None: + return 1.0 + distance_km = haversine_km(question_coords, coords) + return distance_multiplier(distance_km, weight=self.weight, decay_km=self.decay_km) diff --git a/src/shared/geo_boost.py b/src/shared/geo_boost.py new file mode 100644 index 0000000..a16b06c --- /dev/null +++ b/src/shared/geo_boost.py @@ -0,0 +1,24 @@ +from __future__ import annotations + +import math + +from src.shared.geocode import Coordinates + +__all__ = ["distance_multiplier", "haversine_km"] + + +def haversine_km(a: Coordinates, b: Coordinates) -> float: + earth_radius_km = 6371.0 + lat1, lon1, lat2, lon2 = ( + math.radians(value) + for value in (a.latitude, a.longitude, b.latitude, b.longitude) + ) + haversine = ( + math.sin((lat2 - lat1) / 2) ** 2 + + math.cos(lat1) * math.cos(lat2) * math.sin((lon2 - lon1) / 2) ** 2 + ) + return 2 * earth_radius_km * math.asin(math.sqrt(haversine)) + + +def distance_multiplier(distance_km: float, *, weight: float, decay_km: float) -> float: + return (1 - weight) + weight * math.exp(-distance_km / decay_km) diff --git a/src/shared/geocode.py b/src/shared/geocode.py new file mode 100644 index 0000000..5784a8a --- /dev/null +++ b/src/shared/geocode.py @@ -0,0 +1,94 @@ +from __future__ import annotations + +import time +from dataclasses import dataclass, field +from typing import NamedTuple, Protocol + +import httpx +from pydantic import BaseModel, Field + +from src.shared.llm import StructuredLlm + +__all__ = [ + "Coordinates", + "ExtractedLocation", + "GeocodeProvider", + "NominatimGeocoder", + "extract_location_query", + "locate_text", +] + +_LOCATION_PROMPT = ( + "Identify the single specific real-world place this text is about.\n" + "Reply with a short, geocodable name such as 'Brestanica Castle, Slovenia', " + "or null if the text does not name or clearly imply one specific place.\n\n" + "text:\n{text}" +) + + +class Coordinates(NamedTuple): + latitude: float + longitude: float + + +class ExtractedLocation(BaseModel): + location_query: str | None = Field( + default=None, + description=( + "A short, geocodable place name mentioned or clearly implied by " + "the text, or null if no single specific place is identifiable." + ), + ) + + +class GeocodeProvider(Protocol): + def geocode(self, query: str) -> Coordinates | None: ... + + +@dataclass +class NominatimGeocoder: + user_agent: str = "llms4eu-tourism-rag" + base_url: str = "https://nominatim.openstreetmap.org/search" + min_interval_seconds: float = 1.0 + _last_request_at: float = field(default=0.0, init=False, repr=False) + + def geocode(self, query: str) -> Coordinates | None: + self._throttle() + try: + response = httpx.get( + self.base_url, + params={"q": query, "format": "json", "limit": 1}, + headers={"User-Agent": self.user_agent}, + timeout=10.0, + ) + except httpx.RequestError: + return None + if response.status_code != 200: + return None + results = response.json() + if not results: + return None + return Coordinates(float(results[0]["lat"]), float(results[0]["lon"])) + + def _throttle(self) -> None: + elapsed = time.monotonic() - self._last_request_at + if elapsed < self.min_interval_seconds: + time.sleep(self.min_interval_seconds - elapsed) + self._last_request_at = time.monotonic() + + +def extract_location_query(llm: StructuredLlm, text: str) -> str | None: + result = llm.structured_output( + _LOCATION_PROMPT.format(text=text), ExtractedLocation + ) + query = (result.location_query or "").strip() + return query or None + + +def locate_text( + llm: StructuredLlm, geocoder: GeocodeProvider, text: str +) -> Coordinates | None: + query = extract_location_query(llm, text) + if query is None: + return None + return geocoder.geocode(query) diff --git a/tests/test_shared_geo_boost.py b/tests/test_shared_geo_boost.py new file mode 100644 index 0000000..e26a0a5 --- /dev/null +++ b/tests/test_shared_geo_boost.py @@ -0,0 +1,17 @@ +from src.shared.geo_boost import distance_multiplier, haversine_km +from src.shared.geocode import Coordinates + + +def test_haversine_km_matches_known_distance(): + ljubljana = Coordinates(46.0569, 14.5058) + zagreb = Coordinates(45.8150, 15.9819) + assert 110 < haversine_km(ljubljana, zagreb) < 125 + + +def test_distance_multiplier_is_one_at_zero_distance(): + assert distance_multiplier(0, weight=0.25, decay_km=50) == 1.0 + + +def test_distance_multiplier_floors_near_one_minus_weight_far_away(): + far = distance_multiplier(100_000, weight=0.25, decay_km=50) + assert 0.75 <= far < 0.751