From a6b33254aba9be9c6c6444b1bbc581bd331c0463 Mon Sep 17 00:00:00 2001
From: David Riccitelli <david@wordlift.io>
Date: Thu, 24 Sep 2026 14:47:54 -0400
Subject: [PATCH] fix(ingestion): normalize mixed sitemap lastmod values

---
 tests/ingestion/test_sources.py               | 29 +++++++++++++++++++
 .../url_provider/test_sitemap_url_provider.py | 25 ++++++++++++++++
 wordlift_sdk/ingestion/sources.py             |  2 +-
 wordlift_sdk/url_source/sitemap_url_source.py |  2 +-
 4 files changed, 56 insertions(+), 2 deletions(-)

diff --git a/tests/ingestion/test_sources.py b/tests/ingestion/test_sources.py
index c11b6ff..824008e 100644
--- a/tests/ingestion/test_sources.py
+++ b/tests/ingestion/test_sources.py
@@ -87,6 +87,35 @@ def _mock_sitemap_to_df(*, sitemap_url: str, request_headers: dict[str, str]):
     ]
 
 
+def test_sitemap_source_adapter_normalizes_mixed_w3c_lastmod(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    adapter = SitemapSourceAdapter()
+    monkeypatch.setattr(
+        "wordlift_sdk.ingestion.sources.adv.sitemaps.sitemap_to_df",
+        lambda *, sitemap_url, request_headers: pd.DataFrame(
+            {
+                "loc": [
+                    "https://example.com/date-only",
+                    "https://example.com/full-timestamp",
+                ],
+                "lastmod": ["2026-09-22", "2024-10-24T12:20:57+00:00"],
+            }
+        ),
+    )
+
+    items = list(
+        adapter.iter_items(
+            _config(source_config={"sitemap_url": "https://example.com/sitemap.xml"})
+        )
+    )
+
+    assert [item.metadata["date_modified"] for item in items] == [
+        "2026-09-22T00:00:00+00:00",
+        "2024-10-24T12:20:57+00:00",
+    ]
+
+
 def test_sitemap_source_adapter_handles_missing_lastmod_and_failures(
     monkeypatch: pytest.MonkeyPatch,
 ) -> None:
diff --git a/tests/url_provider/test_sitemap_url_provider.py b/tests/url_provider/test_sitemap_url_provider.py
index e6da483..ae037fe 100644
--- a/tests/url_provider/test_sitemap_url_provider.py
+++ b/tests/url_provider/test_sitemap_url_provider.py
@@ -1,6 +1,7 @@
 import logging
 import re
 
+import pandas as pd
 import pytest
 
 from wordlift_sdk.url_source import UrlSource, SitemapUrlSource
@@ -30,3 +31,27 @@ async def test(sitemap_url_provider: UrlSource) -> None:
         urls.append(url)
 
     assert len(urls) == 3565
+
+
+@pytest.mark.asyncio
+async def test_mixed_lastmod_formats(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr(
+        "wordlift_sdk.url_source.sitemap_url_source.adv.sitemaps.sitemap_to_df",
+        lambda *, sitemap_url, request_headers: pd.DataFrame(
+            {
+                "loc": [
+                    "https://example.com/date-only",
+                    "https://example.com/full-timestamp",
+                ],
+                "lastmod": ["2026-09-22", "2024-10-24T12:20:57+00:00"],
+            }
+        ),
+    )
+    source = SitemapUrlSource("https://example.com/sitemap.xml")
+
+    urls = [url async for url in source.urls()]
+
+    assert [url.date_modified.isoformat() for url in urls if url.date_modified] == [
+        "2026-09-22T00:00:00+00:00",
+        "2024-10-24T12:20:57+00:00",
+    ]
diff --git a/wordlift_sdk/ingestion/sources.py b/wordlift_sdk/ingestion/sources.py
index 62a154a..61dbcb7 100644
--- a/wordlift_sdk/ingestion/sources.py
+++ b/wordlift_sdk/ingestion/sources.py
@@ -59,7 +59,7 @@ def iter_items(self, config: ResolvedIngestionConfig) -> Iterator[SourceItem]:
         if "lastmod" not in sitemap_df.columns:
             sitemap_df["lastmod"] = None
         sitemap_df["lastmod_as_datetime"] = pd.to_datetime(
-            sitemap_df["lastmod"], errors="coerce"
+            sitemap_df["lastmod"], errors="coerce", format="mixed", utc=True
         )
 
         for idx, row in sitemap_df.iterrows():
diff --git a/wordlift_sdk/url_source/sitemap_url_source.py b/wordlift_sdk/url_source/sitemap_url_source.py
index a8bf9e4..bbe560d 100644
--- a/wordlift_sdk/url_source/sitemap_url_source.py
+++ b/wordlift_sdk/url_source/sitemap_url_source.py
@@ -26,7 +26,7 @@ async def urls(self) -> AsyncGenerator[Url, None]:
         if "lastmod" not in sitemap_df.columns:
             sitemap_df["lastmod"] = None
         sitemap_df["lastmod_as_datetime"] = pd.to_datetime(
-            sitemap_df["lastmod"], errors="coerce"
+            sitemap_df["lastmod"], errors="coerce", format="mixed", utc=True
         )
 
         for _, row in sitemap_df.iterrows():