seo.py

#!/usr/bin/env python3
"""SEO post-processor for research.ovr.ai.

The publisher emits self-contained, head-less HTML: no <!DOCTYPE>, no <head>,
images and multi-MB viewer data inlined. That is convenient to ship but bad for
search: no title/description/canonical/OG on posts, and 4-15 MB documents
(Googlebot indexes only the first 15 MB of an HTML file).

This script fixes the published output in place, so it must run after every
publish. It is idempotent: running it twice changes nothing.

  1. Moves inline base64 images and big inline <script> blocks into
     content-addressed files under assets/ (cacheable, shared across posts).
  2. Wraps every page in a proper document with a generated <head>
     (title, description, canonical, Open Graph, Twitter, JSON-LD).
  3. Regenerates sitemap.xml, robots.txt, feed.xml and llms.txt from index.json.

Usage: python3 scripts/seo.py   (stdlib only, run from anywhere)
"""
import base64
import hashlib
import html
import json
import re
from datetime import datetime
from email.utils import format_datetime
from pathlib import Path

ROOT = Path(__file__).resolve().parent.parent
SITE = 'https://research.ovr.ai'
SITE_NAME = 'OVER Research'
HOME_TITLE = 'OVER Research — Open Science for Spatial Intelligence and Physical AI'
HOME_DESCRIPTION = (
    'Research notes from OVER on spatial intelligence and physical AI: 3D reconstruction, '
    'Gaussian splatting and geospatial models, with open code, models and data.'
)
TWITTER = '@OVRtheReality'
ORG = {
    '@type': 'Organization',
    'name': 'OVER',
    'url': 'https://www.overthereality.ai',
    'sameAs': [
        'https://github.com/OVR-Platform',
        'https://huggingface.co/OverTheReality',
        'https://twitter.com/OVRtheReality',
    ],
}
# Open dataset described on the homepage; exposed as schema.org Dataset so it
# can surface in Google Dataset Search.
DATASET = {
    '@type': 'Dataset',
    'name': 'OverMaps-1k',
    'description': (
        'An open-source sample of the OVER 3D Maps Dataset: 1,000 real-world 3D maps with '
        'multi-view images, depth and metric scale, for benchmarking visual relocalization, '
        'depth estimation and 3D reconstruction models.'
    ),
    'url': 'https://huggingface.co/datasets/OverTheReality/OverMaps_1k',
    'sameAs': 'https://ovr-platform.github.io/OVR-MAPS/',
    'creator': ORG,
}

# Inline scripts above this size are moved out (three.js, viewer point clouds).
SCRIPT_THRESHOLD = 50 * 1024
IMG_EXT = {'image/png': 'png', 'image/jpeg': 'jpg', 'image/webp': 'webp',
           'image/gif': 'gif', 'image/svg+xml': 'svg'}

SEO_START = '<!-- seo:start (generated by scripts/seo.py, do not edit) -->'
SEO_END = '<!-- seo:end -->'
BODY_OPEN = f'{SEO_END}\n</head>\n<body>\n'
BODY_CLOSE = '\n</body>\n</html>\n'

IMG_SRC_RE = re.compile(r'''src=(["'])data:(image/[a-z+]+);base64,([A-Za-z0-9+/=]+)\1''')
SCRIPT_RE = re.compile(r'<script>(.*?)</script>', re.S)
# Tags the publisher puts at the top of the homepage; replaced by the generated head.
LEGACY_HEAD_RE = re.compile(r'<title>.*?</title>|<meta\s+(?:name="(?:viewport|description)"|charset=|http-equiv=)[^>]*>\s*', re.S)


def write_if_changed (path, data):
    """Write only on change so reruns leave mtimes and git status untouched."""
    mode = 'wb' if isinstance(data, bytes) else 'w'
    current = path.read_bytes() if path.exists() else None
    new = data if isinstance(data, bytes) else data.encode('utf-8')
    if current != new:
        path.parent.mkdir(parents=True, exist_ok=True)
        with open(path, mode, **({} if mode == 'wb' else {'encoding': 'utf-8'})) as f:
            f.write(data)


def store_asset (data, kind, ext):
    """Content-addressed name: identical blobs (logos, three.js) dedupe across pages."""
    name = f'assets/{kind}/{hashlib.sha256(data).hexdigest()[:12]}.{ext}'
    write_if_changed(ROOT / name, data)
    return name


def unwrap (doc):
    """Undo a previous run so the page is always rebuilt from the publisher's body."""
    if doc.startswith('<!DOCTYPE html>') and BODY_OPEN in doc:
        doc = doc.split(BODY_OPEN, 1)[1]
        doc = doc[:-len(BODY_CLOSE)] if doc.endswith(BODY_CLOSE) else doc
    first_style = doc.find('<style')
    if first_style > 0:
        doc = LEGACY_HEAD_RE.sub('', doc[:first_style]) + doc[first_style:]
    return doc


def externalize (body, prefix):
    """Move base64 images and big inline scripts out of the HTML. Returns (body, image paths)."""
    images = []

    def img_repl (m):
        quote, mime, b64 = m.groups()
        path = store_asset(base64.b64decode(b64), 'img', IMG_EXT.get(mime, 'bin'))
        images.append(path)
        return f'src={quote}{prefix}{path}{quote}'

    def script_repl (m):
        code = m.group(1)
        if len(code) < SCRIPT_THRESHOLD:
            return m.group(0)
        # Classic external script keeps inline semantics: sync, global scope, same order.
        return f'<script src="{prefix}{store_asset(code.encode("utf-8"), "js", "js")}"></script>'

    body = IMG_SRC_RE.sub(img_repl, body)
    body = SCRIPT_RE.sub(script_repl, body)
    # Already-externalized images from a previous run still count for cover detection.
    images += re.findall(r'src=["\'](?:\.\./)*(assets/img/[^"\']+)', body)
    return lazy_images(body), list(dict.fromkeys(images))


def lazy_images (body):
    """Lazy-load every image except the first (header logo, above the fold)."""
    count = 0

    def repl (m):
        nonlocal count
        count += 1
        tag = m.group(0)
        if count == 1 or 'loading=' in tag:
            return tag
        return tag[:4] + ' loading="lazy" decoding="async"' + tag[4:]

    return re.sub(r'<img\b[^>]*>', repl, body)


def meta_description (text, limit=160):
    """Whole sentences up to ~160 chars; Google truncates longer snippets anyway."""
    out = ''
    for sentence in re.split(r'(?<=[.!?])\s+', text.strip()):
        if len(out) + len(sentence) + 1 > limit:
            break
        out = f'{out} {sentence}'.strip()
    if not out:
        out = text[:limit - 1].rsplit(' ', 1)[0].rstrip(',;:—–- ') + '…'
    return out


def json_ld (data):
    # '</' inside JSON would close the <script> element early.
    return json.dumps({'@context': 'https://schema.org', **data}, ensure_ascii=False).replace('</', '<\\/')


def build_head (prefix, title, description, url, og_type, image, extra_meta='', ld=None):
    e = html.escape
    tags = [
        '<meta charset="utf-8">',
        '<meta name="viewport" content="width=device-width,initial-scale=1">',
        f'<title>{e(title)}</title>',
        f'<meta name="description" content="{e(description)}">',
        f'<link rel="canonical" href="{e(url)}">',
        f'<link rel="icon" href="{prefix}favicon.ico">',
        f'<link rel="alternate" type="application/rss+xml" title="{SITE_NAME}" href="{SITE}/feed.xml">',
        f'<meta property="og:site_name" content="{SITE_NAME}">',
        f'<meta property="og:type" content="{og_type}">',
        f'<meta property="og:title" content="{e(title)}">',
        f'<meta property="og:description" content="{e(description)}">',
        f'<meta property="og:url" content="{e(url)}">',
        '<meta name="twitter:card" content="summary_large_image">',
        f'<meta name="twitter:site" content="{TWITTER}">',
        f'<meta name="twitter:title" content="{e(title)}">',
        f'<meta name="twitter:description" content="{e(description)}">',
    ]
    if image:
        tags += [f'<meta property="og:image" content="{e(image)}">',
                 f'<meta name="twitter:image" content="{e(image)}">']
    tags.append(extra_meta)
    tags += [f'<script type="application/ld+json">{json_ld(x)}</script>' for x in ld or []]
    head = '\n'.join(t for t in tags if t)
    return f'<!DOCTYPE html>\n<html lang="en">\n<head>\n{SEO_START}\n{head}\n{BODY_OPEN}'


def pick_cover (post, images, chrome):
    """Explicit card.json "cover" wins; else the first figure that isn't site chrome."""
    if post.get('cover'):
        return f"{post['post_url'].rstrip('/')}/{post['cover']}"
    content = [p for p in images if p not in chrome and not p.endswith('.svg')]
    return f'{SITE}/{content[0]}' if content else None


def process_post (post, chrome):
    page = ROOT / 'posts' / post['slug'] / 'index.html'
    body, images = externalize(unwrap(page.read_text(encoding='utf-8')), '../../')
    cover = pick_cover(post, images, chrome)
    description = meta_description(post['description'])
    title = f"{post['title']} · {SITE_NAME}"
    article = {
        '@type': 'ScholarlyArticle',
        'headline': post['title'],
        'alternativeHeadline': post.get('subtitle'),
        'description': post['description'],
        'datePublished': post['date'],
        'url': post['post_url'],
        'mainEntityOfPage': post['post_url'],
        'keywords': post.get('tags', []),
        'author': {'@type': 'Organization', 'name': post.get('author', SITE_NAME)},
        'publisher': ORG,
        'isPartOf': {'@type': 'Blog', 'name': SITE_NAME, 'url': SITE + '/'},
        'isBasedOn': [r['url'] for r in post.get('resources', [])] or None,
    }
    if cover:
        article['image'] = cover
    article = {k: v for k, v in article.items() if v}
    extra = '\n'.join([f'<meta property="article:published_time" content="{post["date"]}">'] +
                      [f'<meta property="article:tag" content="{html.escape(t)}">' for t in post.get('tags', [])])
    head = build_head('../../', title, description, post['post_url'], 'article', cover, extra, [article])
    write_if_changed(page, head + body + BODY_CLOSE)
    post['_cover'] = cover
    return body


def process_home (posts):
    page = ROOT / 'index.html'
    body, images = externalize(unwrap(page.read_text(encoding='utf-8')), '')
    cover = next((p['_cover'] for p in posts if p.get('_cover')), None)
    blog = {
        '@type': 'Blog',
        'name': SITE_NAME,
        'url': SITE + '/',
        'description': HOME_DESCRIPTION,
        'publisher': ORG,
        'blogPost': [{'@type': 'ScholarlyArticle', 'headline': p['title'], 'url': p['post_url'],
                      'datePublished': p['date']} for p in posts],
    }
    head = build_head('', HOME_TITLE, HOME_DESCRIPTION, SITE + '/', 'website', cover, ld=[blog, DATASET])
    write_if_changed(page, head + body + BODY_CLOSE)
    return body


def write_sitemap (posts):
    urls = [(SITE + '/', max(p['date'] for p in posts))] + [(p['post_url'], p['date']) for p in posts]
    items = ''.join(f'  <url><loc>{html.escape(u)}</loc><lastmod>{d}</lastmod></url>\n' for u, d in urls)
    write_if_changed(ROOT / 'sitemap.xml', '<?xml version="1.0" encoding="UTF-8"?>\n'
                     '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n' + items + '</urlset>\n')
    write_if_changed(ROOT / 'robots.txt', f'User-agent: *\nAllow: /\n\nSitemap: {SITE}/sitemap.xml\n')


def write_feed (posts):
    e = html.escape

    def rfc822 (day):
        return format_datetime(datetime.fromisoformat(day + 'T00:00:00+00:00'))

    items = ''.join(
        '  <item>\n'
        f'    <title>{e(p["title"])}</title>\n'
        f'    <link>{e(p["post_url"])}</link>\n'
        f'    <guid isPermaLink="true">{e(p["post_url"])}</guid>\n'
        f'    <pubDate>{rfc822(p["date"])}</pubDate>\n'
        f'    <description>{e(p["description"])}</description>\n'
        + ''.join(f'    <category>{e(t)}</category>\n' for t in p.get('tags', []))
        + '  </item>\n'
        for p in posts)
    write_if_changed(ROOT / 'feed.xml', '<?xml version="1.0" encoding="UTF-8"?>\n'
                     '<rss version="2.0" xmlns:atom="http://www.w3.org/2005/Atom">\n<channel>\n'
                     f'  <title>{SITE_NAME}</title>\n  <link>{SITE}/</link>\n'
                     f'  <description>{e(HOME_DESCRIPTION)}</description>\n  <language>en</language>\n'
                     f'  <lastBuildDate>{rfc822(max(p["date"] for p in posts))}</lastBuildDate>\n'
                     f'  <atom:link href="{SITE}/feed.xml" rel="self" type="application/rss+xml"/>\n'
                     + items + '</channel>\n</rss>\n')


def write_llms_txt (posts):
    """llms.txt: plain-text map of the site for AI search/answer engines."""
    lines = [f'# {SITE_NAME}', '', f'> {HOME_DESCRIPTION}', '', '## Research notes', '']
    lines += [f'- [{p["title"]}]({p["post_url"]}): {p["description"]}' for p in posts]
    lines += ['', '## Data and code', '',
              f'- [OverMaps-1k]({DATASET["url"]}): {DATASET["description"]}',
              '- [GitHub — OVR-Platform](https://github.com/OVR-Platform)',
              '- [Hugging Face — OverTheReality](https://huggingface.co/OverTheReality)', '']
    write_if_changed(ROOT / 'llms.txt', '\n'.join(lines))


def prune_assets (bodies):
    """Drop assets no page references any more (content changed → new hash)."""
    used = set(re.findall(r'assets/(?:img|js)/[0-9a-f]{12}\.\w+', ''.join(bodies)))
    for f in (ROOT / 'assets').glob('*/*'):
        if f.relative_to(ROOT).as_posix() not in used:
            f.unlink()


def main ():
    # Entries without post_url are external cards (datasets, reports) with no page here.
    entries = json.loads((ROOT / 'index.json').read_text(encoding='utf-8'))['posts']
    posts = sorted((p for p in entries if p.get('post_url')), key=lambda p: p['date'], reverse=True)
    unlisted = {d.name for d in (ROOT / 'posts').iterdir() if d.is_dir()} - {p['slug'] for p in posts}
    if unlisted:
        print(f'warning: posts missing from index.json (not processed): {sorted(unlisted)}')

    # Images already on the homepage (logos, EU banner) are chrome, never a post cover.
    _, chrome = externalize(unwrap((ROOT / 'index.html').read_text(encoding='utf-8')), '')
    bodies = [process_post(p, set(chrome)) for p in posts]
    bodies.append(process_home(posts))
    write_sitemap(posts)
    write_feed(posts)
    write_llms_txt(posts)
    prune_assets(bodies)

    for p in [ROOT / 'index.html'] + [ROOT / 'posts' / p['slug'] / 'index.html' for p in posts]:
        print(f'{p.relative_to(ROOT)}: {p.stat().st_size // 1024} KB')


if __name__ == '__main__':
    main()