From c1c1f0b32e232c56825bf95833279a33514d69ba Mon Sep 17 00:00:00 2001 From: Nilesh Raj Date: Thu, 22 Oct 2026 14:59:17 +0530 Subject: [PATCH] Update crawl and indexing directives --- app/about/page.tsx | 13 +++++---- app/account/page.tsx | 2 +- app/agents/[slug]/page.tsx | 3 +- app/categories/[slug]/page.tsx | 7 ++++- app/contact/layout.tsx | 13 +++++---- app/cookies/page.tsx | 13 +++++---- app/faq/page.tsx | 13 +++++---- app/join/page.tsx | 13 +++++---- app/login/page.tsx | 2 +- app/methodology/page.tsx | 13 +++++---- app/news/[slug]/page.tsx | 1 + app/privacy/page.tsx | 13 +++++---- app/profile/page.tsx | 2 +- app/signup/page.tsx | 2 +- app/sitemap.ts | 19 ------------- app/terms/page.tsx | 13 +++++---- lib/seo.ts | 3 ++ lib/sitemap.ts | 8 ------ tests/sitemap-generation.test.ts | 33 ++++++++++++---------- tests/static-indexing-policy.test.ts | 42 ++++++++++++++++++++++++++++ 20 files changed, 141 insertions(+), 87 deletions(-) create mode 100644 tests/static-indexing-policy.test.ts diff --git a/app/about/page.tsx b/app/about/page.tsx index 1a6569c..d061a98 100644 --- a/app/about/page.tsx +++ b/app/about/page.tsx @@ -68,11 +68,14 @@ const teamMembers = [ }, ] satisfies readonly TeamRevealMember[]; -export const metadata = createPublicPageMetadata({ - title: "About", - description: "How AgentNine selects, reviews, and maintains AI agent project listings.", - path: "/about", -}); +export const metadata = { + ...createPublicPageMetadata({ + title: "About", + description: "How AgentNine selects, reviews, and maintains AI agent project listings.", + path: "/about", + }), + robots: { index: false, follow: true }, +}; export default function AboutPage() { const jsonLd = { diff --git a/app/account/page.tsx b/app/account/page.tsx index e184bc0..fa5416b 100644 --- a/app/account/page.tsx +++ b/app/account/page.tsx @@ -1,7 +1,7 @@ import { AccountDashboard } from "@/components/AccountDashboard"; export const dynamic = "force-dynamic"; -export const metadata = { title: "Your account", robots: { index: false, follow: false } }; +export const metadata = { title: "Your account", robots: { index: false, follow: true } }; export default async function AccountPage() { return

Account

Saved agents.

; diff --git a/app/agents/[slug]/page.tsx b/app/agents/[slug]/page.tsx index a30ea4c..24473a3 100644 --- a/app/agents/[slug]/page.tsx +++ b/app/agents/[slug]/page.tsx @@ -31,13 +31,14 @@ export async function generateStaticParams(): Promise<{ slug: string }[]> { export async function generateMetadata({ params }: { params: Promise<{ slug: string }> }): Promise { const agent = await getAgent((await params).slug); - if (!agent) return { title: "Agent not found" }; + if (!agent) return { title: "Agent not found", robots: { index: false, follow: false } }; const description = `Review ${agent.name}, an open-source AI agent. Check its setup guide, source repository, system requirements, and documented access. ${agent.short_description}`; return createPublicPageMetadata({ title: `${agent.name} AI Agent Setup Guide`, description, path: `/agents/${agent.slug}`, type: "article", + robots: { index: true, follow: true }, }); } diff --git a/app/categories/[slug]/page.tsx b/app/categories/[slug]/page.tsx index e776feb..3f0d04e 100644 --- a/app/categories/[slug]/page.tsx +++ b/app/categories/[slug]/page.tsx @@ -21,7 +21,12 @@ export async function generateStaticParams(): Promise<{ slug: string }[]> { export async function generateMetadata({ params }: { params: Promise<{ slug: string }> }) { const { slug } = await params; const category = await getCategory(slug); - return createPublicPageMetadata({ ...getCategoryMetadata(category), path: `/categories/${slug}` }); + if (!category) return { title: "Category not found", robots: { index: false, follow: false } }; + return createPublicPageMetadata({ + ...getCategoryMetadata(category), + path: `/categories/${slug}`, + robots: { index: true, follow: true }, + }); } export default async function CategoryPage({ params }: { params: Promise<{ slug: string }> }) { diff --git a/app/contact/layout.tsx b/app/contact/layout.tsx index 5703655..3b846e1 100644 --- a/app/contact/layout.tsx +++ b/app/contact/layout.tsx @@ -1,11 +1,14 @@ import type { Metadata } from "next"; import { createPublicPageMetadata } from "@/lib/seo"; -export const metadata: Metadata = createPublicPageMetadata({ - title: "Contact", - description: "Suggest an AI agent, report a listing issue, or send a correction to AgentNine.", - path: "/contact", -}); +export const metadata: Metadata = { + ...createPublicPageMetadata({ + title: "Contact", + description: "Suggest an AI agent, report a listing issue, or send a correction to AgentNine.", + path: "/contact", + }), + robots: { index: false, follow: true }, +}; export default function ContactLayout({ children }: Readonly<{ children: React.ReactNode }>) { return children; diff --git a/app/cookies/page.tsx b/app/cookies/page.tsx index f5e9b06..3a066e4 100644 --- a/app/cookies/page.tsx +++ b/app/cookies/page.tsx @@ -2,11 +2,14 @@ import Link from "next/link"; import { createPublicPageMetadata } from "@/lib/seo"; import { legalConfig } from "@/lib/legal-config"; -export const metadata = createPublicPageMetadata({ - title: "Cookies and preferences", - description: "How AgentNine uses necessary storage and optional analytics, session replay, and marketing services.", - path: "/cookies", -}); +export const metadata = { + ...createPublicPageMetadata({ + title: "Cookies and preferences", + description: "How AgentNine uses necessary storage and optional analytics, session replay, and marketing services.", + path: "/cookies", + }), + robots: { index: false, follow: true }, +}; export default function CookiesPage() { return ( diff --git a/app/faq/page.tsx b/app/faq/page.tsx index 6638cc0..4958df5 100644 --- a/app/faq/page.tsx +++ b/app/faq/page.tsx @@ -4,11 +4,14 @@ import { faqGroups } from "@/lib/faq"; import { serializeJsonLd } from "@/lib/json-ld"; import { createPublicPageMetadata } from "@/lib/seo"; -export const metadata = createPublicPageMetadata({ - title: "AI agent directory FAQ: setup and verification", - description: "Answers about AI-agent listings, setup guides, source details, recorded checks, and what AgentNine verification marks mean.", - path: "/faq", -}); +export const metadata = { + ...createPublicPageMetadata({ + title: "AI agent directory FAQ: setup and verification", + description: "Answers about AI-agent listings, setup guides, source details, recorded checks, and what AgentNine verification marks mean.", + path: "/faq", + }), + robots: { index: false, follow: true }, +}; export default function FaqPage() { const jsonLd = { diff --git a/app/join/page.tsx b/app/join/page.tsx index 7df28de..3ca2447 100644 --- a/app/join/page.tsx +++ b/app/join/page.tsx @@ -2,11 +2,14 @@ import type { Metadata } from "next"; import { JoinForm } from "@/components/JoinForm"; import { createPublicPageMetadata } from "@/lib/seo"; -export const metadata: Metadata = createPublicPageMetadata({ - title: "Join AgentNine", - description: "Propose a contribution to AgentNine’s AI agent directory.", - path: "/join", -}); +export const metadata: Metadata = { + ...createPublicPageMetadata({ + title: "Join AgentNine", + description: "Propose a contribution to AgentNine’s AI agent directory.", + path: "/join", + }), + robots: { index: false, follow: true }, +}; export default function JoinPage() { return ( diff --git a/app/login/page.tsx b/app/login/page.tsx index c328cb9..0b9c2ad 100644 --- a/app/login/page.tsx +++ b/app/login/page.tsx @@ -1,7 +1,7 @@ import { AuthForm } from "@/components/AuthForm"; import { AuthPageShell } from "@/components/AuthPageShell"; -export const metadata = { title: "Sign in", robots: { index: false, follow: false } }; +export const metadata = { title: "Sign in", robots: { index: false, follow: true } }; export default async function LoginPage({ searchParams }: { searchParams: Promise<{ authError?: string }> }) { const { authError } = await searchParams; diff --git a/app/methodology/page.tsx b/app/methodology/page.tsx index 1088f32..e6f28d8 100644 --- a/app/methodology/page.tsx +++ b/app/methodology/page.tsx @@ -3,11 +3,14 @@ import Link from "next/link"; import ArrowRight from "reicon-react/icons/ArrowRight"; import { createPublicPageMetadata } from "@/lib/seo"; -export const metadata: Metadata = createPublicPageMetadata({ - title: "Trust and methodology", - description: "How AgentNine records verification evidence, freshness, and uncertainty.", - path: "/methodology", -}); +export const metadata: Metadata = { + ...createPublicPageMetadata({ + title: "Trust and methodology", + description: "How AgentNine records verification evidence, freshness, and uncertainty.", + path: "/methodology", + }), + robots: { index: false, follow: true }, +}; export default function MethodologyPage() { return ( diff --git a/app/news/[slug]/page.tsx b/app/news/[slug]/page.tsx index 4357385..965b0e8 100644 --- a/app/news/[slug]/page.tsx +++ b/app/news/[slug]/page.tsx @@ -25,6 +25,7 @@ export async function generateMetadata({ params }: { params: Promise<{ slug: str description: post.summary, path: `/news/${post.slug}`, type: "article", + robots: { index: true, follow: true }, }); } diff --git a/app/privacy/page.tsx b/app/privacy/page.tsx index 32cea22..62d3c0f 100644 --- a/app/privacy/page.tsx +++ b/app/privacy/page.tsx @@ -2,11 +2,14 @@ import Link from "next/link"; import { createPublicPageMetadata } from "@/lib/seo"; import { legalConfig } from "@/lib/legal-config"; -export const metadata = createPublicPageMetadata({ - title: "Privacy policy", - description: "How AgentNine handles account, contact, feedback, preference, analytics, and session replay information.", - path: "/privacy", -}); +export const metadata = { + ...createPublicPageMetadata({ + title: "Privacy policy", + description: "How AgentNine handles account, contact, feedback, preference, analytics, and session replay information.", + path: "/privacy", + }), + robots: { index: false, follow: true }, +}; const processorRows = [ { provider: "Supabase", purpose: "Database, authentication, preferences, feedback, and account data" }, diff --git a/app/profile/page.tsx b/app/profile/page.tsx index 45ae361..f35da16 100644 --- a/app/profile/page.tsx +++ b/app/profile/page.tsx @@ -1,7 +1,7 @@ import { ProfileDetails } from "@/components/ProfileDetails"; export const dynamic = "force-dynamic"; -export const metadata = { title: "Your profile", robots: { index: false, follow: false } }; +export const metadata = { title: "Your profile", robots: { index: false, follow: true } }; export default function ProfilePage() { return ( diff --git a/app/signup/page.tsx b/app/signup/page.tsx index 282f5ac..be01f89 100644 --- a/app/signup/page.tsx +++ b/app/signup/page.tsx @@ -1,7 +1,7 @@ import { AuthForm } from "@/components/AuthForm"; import { AuthPageShell } from "@/components/AuthPageShell"; -export const metadata = { title: "Create an account", robots: { index: false, follow: false } }; +export const metadata = { title: "Create an account", robots: { index: false, follow: true } }; export default async function SignupPage({ searchParams }: { searchParams: Promise<{ authError?: string }> }) { const { authError } = await searchParams; diff --git a/app/sitemap.ts b/app/sitemap.ts index 8df8858..9f11abc 100644 --- a/app/sitemap.ts +++ b/app/sitemap.ts @@ -4,10 +4,8 @@ import { getPublishedNewsPosts } from "@/lib/news"; import { getLatestSitemapLastModified, getSitemapLastModified, - STATIC_PAGE_LAST_MODIFIED, } from "@/lib/sitemap"; import { getLatestPublishedSiteFeedbackUpdate } from "@/lib/site-feedback-data"; -import { legalConfig } from "@/lib/legal-config"; import { SITE_URL } from "@/lib/site"; export const dynamic = "force-dynamic"; @@ -45,28 +43,11 @@ export default async function sitemap(): Promise { const latestNewsUpdate = getLatestSitemapLastModified( newsPosts.map((post) => post.updated_at ?? post.published_at ?? post.created_at), ); - const legalLastModified = getSitemapLastModified(legalConfig.lastUpdated); - const staticPageLastModified = { - about: getSitemapLastModified(STATIC_PAGE_LAST_MODIFIED.about), - faq: getSitemapLastModified(STATIC_PAGE_LAST_MODIFIED.faq), - join: getSitemapLastModified(STATIC_PAGE_LAST_MODIFIED.join), - contact: getSitemapLastModified(STATIC_PAGE_LAST_MODIFIED.contact), - methodology: getSitemapLastModified(STATIC_PAGE_LAST_MODIFIED.methodology), - }; - return [ { url: base, ...(latestHomeUpdate ? { lastModified: latestHomeUpdate } : {}), changeFrequency: "weekly", priority: 1 }, { url: `${base}/agents`, ...(latestAgentUpdate ? { lastModified: latestAgentUpdate } : {}), changeFrequency: "weekly", priority: 0.8 }, { url: `${base}/news`, ...(latestNewsUpdate ? { lastModified: latestNewsUpdate } : {}), changeFrequency: "daily", priority: 0.7 }, { url: `${base}/categories`, ...(latestAgentUpdate ? { lastModified: latestAgentUpdate } : {}), changeFrequency: "weekly", priority: 0.8 }, - { url: `${base}/methodology`, ...(staticPageLastModified.methodology ? { lastModified: staticPageLastModified.methodology } : {}), changeFrequency: "monthly", priority: 0.5 }, - { url: `${base}/join`, ...(staticPageLastModified.join ? { lastModified: staticPageLastModified.join } : {}), changeFrequency: "monthly", priority: 0.4 }, - { url: `${base}/about`, ...(staticPageLastModified.about ? { lastModified: staticPageLastModified.about } : {}), changeFrequency: "monthly", priority: 0.5 }, - { url: `${base}/faq`, ...(staticPageLastModified.faq ? { lastModified: staticPageLastModified.faq } : {}), changeFrequency: "monthly", priority: 0.5 }, - { url: `${base}/contact`, ...(staticPageLastModified.contact ? { lastModified: staticPageLastModified.contact } : {}), changeFrequency: "monthly", priority: 0.3 }, - { url: `${base}/privacy`, ...(legalLastModified ? { lastModified: legalLastModified } : {}), changeFrequency: "yearly", priority: 0.2 }, - { url: `${base}/cookies`, ...(legalLastModified ? { lastModified: legalLastModified } : {}), changeFrequency: "yearly", priority: 0.2 }, - { url: `${base}/terms`, ...(legalLastModified ? { lastModified: legalLastModified } : {}), changeFrequency: "yearly", priority: 0.2 }, ...categories.map((category) => { const categoryLastModified = getLatestSitemapLastModified( agents.filter((agent) => agent.category_id === category.id).map((agent) => agent.updated_at), diff --git a/app/terms/page.tsx b/app/terms/page.tsx index fd9eb50..1b81566 100644 --- a/app/terms/page.tsx +++ b/app/terms/page.tsx @@ -2,11 +2,14 @@ import Link from "next/link"; import { createPublicPageMetadata } from "@/lib/seo"; import { legalConfig } from "@/lib/legal-config"; -export const metadata = createPublicPageMetadata({ - title: "Terms of service", - description: "Terms for using AgentNine and reviewing listed AI-agent projects.", - path: "/terms", -}); +export const metadata = { + ...createPublicPageMetadata({ + title: "Terms of service", + description: "Terms for using AgentNine and reviewing listed AI-agent projects.", + path: "/terms", + }), + robots: { index: false, follow: true }, +}; export default function TermsPage() { const contact = legalConfig.contactEmail diff --git a/lib/seo.ts b/lib/seo.ts index cbcf849..09947d7 100644 --- a/lib/seo.ts +++ b/lib/seo.ts @@ -6,6 +6,7 @@ type PublicPageMetadata = { description: string; path: string; type?: "website" | "article"; + robots?: Metadata["robots"]; }; const DEFAULT_SOCIAL_IMAGE = "/og-image.png"; @@ -44,6 +45,7 @@ export function createPublicPageMetadata({ description, path, type = "website", + robots, }: PublicPageMetadata): Metadata { const canonicalUrl = new URL(path, SITE_URL).toString(); const metaDescription = normalizeDescription(description); @@ -58,6 +60,7 @@ export function createPublicPageMetadata({ return { title, description: metaDescription, + ...(robots ? { robots } : {}), alternates: { canonical: canonicalUrl }, openGraph: { title: socialTitle, diff --git a/lib/sitemap.ts b/lib/sitemap.ts index 2afe17e..744da92 100644 --- a/lib/sitemap.ts +++ b/lib/sitemap.ts @@ -1,11 +1,3 @@ -export const STATIC_PAGE_LAST_MODIFIED = { - about: "2026-10-19", - faq: "2026-10-17", - join: "2026-10-09", - contact: "2026-10-12", - methodology: "2026-10-17", -} as const; - export function getSitemapLastModified(value: string | null | undefined): Date | undefined { if (!value) return undefined; const timestamp = new Date(value).getTime(); diff --git a/tests/sitemap-generation.test.ts b/tests/sitemap-generation.test.ts index 7380079..087375c 100644 --- a/tests/sitemap-generation.test.ts +++ b/tests/sitemap-generation.test.ts @@ -63,25 +63,30 @@ describe("sitemap generation", () => { expect(entries.find((entry) => entry.url.endsWith("/news/second-story"))?.lastModified).toEqual(lastAgentUpdate); }); - it("includes last-modified dates for legal pages from the legal revision date", async () => { + it("excludes pages marked noindex from the sitemap", async () => { const entries = await sitemap(); - const expected = new Date("October 19, 2026"); - for (const path of ["/privacy", "/cookies", "/terms"]) { - expect(entries.find((entry) => entry.url.endsWith(path))?.lastModified).toEqual(expected); + for (const path of [ + "/about", + "/methodology", + "/join", + "/faq", + "/contact", + "/privacy", + "/cookies", + "/terms", + ]) { + expect(entries.find((entry) => entry.url === `https://agentnine.pro${path}`)).toBeUndefined(); } }); - it("includes the last content edit date for the remaining static pages", async () => { + it("keeps published agents, categories, and news articles in the sitemap", async () => { const entries = await sitemap(); - const expectedDates = { - "/about": "2026-10-19", - "/faq": "2026-10-17", - "/join": "2026-10-09", - "/contact": "2026-10-12", - "/methodology": "2026-10-17", - }; - for (const [path, date] of Object.entries(expectedDates)) { - expect(entries.find((entry) => entry.url.endsWith(path))?.lastModified).toEqual(new Date(date)); + for (const path of [ + "/agents/first-agent", + "/categories/first-category", + "/news/first-story", + ]) { + expect(entries.find((entry) => entry.url === `https://agentnine.pro${path}`)).toBeDefined(); } }); }); diff --git a/tests/static-indexing-policy.test.ts b/tests/static-indexing-policy.test.ts new file mode 100644 index 0000000..9624b19 --- /dev/null +++ b/tests/static-indexing-policy.test.ts @@ -0,0 +1,42 @@ +import { describe, expect, it } from "vitest"; +import { metadata as aboutMetadata } from "@/app/about/page"; +import { metadata as methodologyMetadata } from "@/app/methodology/page"; +import { metadata as joinMetadata } from "@/app/join/page"; +import { metadata as faqMetadata } from "@/app/faq/page"; +import { metadata as contactMetadata } from "@/app/contact/layout"; +import { metadata as privacyMetadata } from "@/app/privacy/page"; +import { metadata as cookiesMetadata } from "@/app/cookies/page"; +import { metadata as termsMetadata } from "@/app/terms/page"; +import { metadata as feedbackMetadata } from "@/app/feedback/page"; +import { metadata as privacyChoicesMetadata } from "@/app/privacy/choices/page"; +import { metadata as loginMetadata } from "@/app/login/page"; +import { metadata as signupMetadata } from "@/app/signup/page"; +import { metadata as profileMetadata } from "@/app/profile/page"; +import { metadata as accountMetadata } from "@/app/account/page"; + +describe("static pages excluded from search indexing", () => { + it.each([ + ["/about", aboutMetadata], + ["/methodology", methodologyMetadata], + ["/join", joinMetadata], + ["/faq", faqMetadata], + ["/contact", contactMetadata], + ["/privacy", privacyMetadata], + ["/cookies", cookiesMetadata], + ["/terms", termsMetadata], + ])("keeps %s crawlable but noindex", (_path, metadata) => { + expect(metadata.robots).toEqual({ index: false, follow: true }); + expect(metadata.alternates?.canonical).toBeTruthy(); + }); + + it.each([ + ["/feedback", feedbackMetadata], + ["/privacy/choices", privacyChoicesMetadata], + ["/login", loginMetadata], + ["/signup", signupMetadata], + ["/profile", profileMetadata], + ["/account", accountMetadata], + ])("keeps %s crawlable while excluding it from search", (_path, metadata) => { + expect(metadata.robots).toEqual({ index: false, follow: true }); + }); +});