Update crawl and indexing directives

app/about/page.tsx

@@ -68,11 +68,14 @@ const teamMembers = [
   },
 ] satisfies readonly TeamRevealMember[];
 
-export const metadata = createPublicPageMetadata({
-  title: "About",
-  description: "How AgentNine selects, reviews, and maintains AI agent project listings.",
-  path: "/about",
-});
+export const metadata = {
+  ...createPublicPageMetadata({
+    title: "About",
+    description: "How AgentNine selects, reviews, and maintains AI agent project listings.",
+    path: "/about",
+  }),
+  robots: { index: false, follow: true },
+};
 
 export default function AboutPage() {
   const jsonLd = {

app/account/page.tsx

@@ -1,7 +1,7 @@
 import { AccountDashboard } from "@/components/AccountDashboard";
 
 export const dynamic = "force-dynamic";
-export const metadata = { title: "Your account", robots: { index: false, follow: false } };
+export const metadata = { title: "Your account", robots: { index: false, follow: true } };
 
 export default async function AccountPage() {
   return <main id="main-content" className="page-shell"><div className="container account-page"><div className="page-heading"><p className="eyebrow">Account</p><h1>Saved agents.</h1></div><AccountDashboard /></div></main>;

app/agents/[slug]/page.tsx

@@ -31,13 +31,14 @@ export async function generateStaticParams(): Promise<{ slug: string }[]> {
 
 export async function generateMetadata({ params }: { params: Promise<{ slug: string }> }): Promise<Metadata> {
   const agent = await getAgent((await params).slug);
-  if (!agent) return { title: "Agent not found" };
+  if (!agent) return { title: "Agent not found", robots: { index: false, follow: false } };
   const description = `Review ${agent.name}, an open-source AI agent. Check its setup guide, source repository, system requirements, and documented access. ${agent.short_description}`;
   return createPublicPageMetadata({
     title: `${agent.name} AI Agent Setup Guide`,
     description,
     path: `/agents/${agent.slug}`,
     type: "article",
+    robots: { index: true, follow: true },
   });
 }
 

app/categories/[slug]/page.tsx

@@ -21,7 +21,12 @@ export async function generateStaticParams(): Promise<{ slug: string }[]> {
 export async function generateMetadata({ params }: { params: Promise<{ slug: string }> }) {
   const { slug } = await params;
   const category = await getCategory(slug);
-  return createPublicPageMetadata({ ...getCategoryMetadata(category), path: `/categories/${slug}` });
+  if (!category) return { title: "Category not found", robots: { index: false, follow: false } };
+  return createPublicPageMetadata({
+    ...getCategoryMetadata(category),
+    path: `/categories/${slug}`,
+    robots: { index: true, follow: true },
+  });
 }
 
 export default async function CategoryPage({ params }: { params: Promise<{ slug: string }> }) {

app/contact/layout.tsx

@@ -1,11 +1,14 @@
 import type { Metadata } from "next";
 import { createPublicPageMetadata } from "@/lib/seo";
 
-export const metadata: Metadata = createPublicPageMetadata({
-  title: "Contact",
-  description: "Suggest an AI agent, report a listing issue, or send a correction to AgentNine.",
-  path: "/contact",
-});
+export const metadata: Metadata = {
+  ...createPublicPageMetadata({
+    title: "Contact",
+    description: "Suggest an AI agent, report a listing issue, or send a correction to AgentNine.",
+    path: "/contact",
+  }),
+  robots: { index: false, follow: true },
+};
 
 export default function ContactLayout({ children }: Readonly<{ children: React.ReactNode }>) {
   return children;

app/cookies/page.tsx

@@ -2,11 +2,14 @@ import Link from "next/link";
 import { createPublicPageMetadata } from "@/lib/seo";
 import { legalConfig } from "@/lib/legal-config";
 
-export const metadata = createPublicPageMetadata({
-  title: "Cookies and preferences",
-  description: "How AgentNine uses necessary storage and optional analytics, session replay, and marketing services.",
-  path: "/cookies",
-});
+export const metadata = {
+  ...createPublicPageMetadata({
+    title: "Cookies and preferences",
+    description: "How AgentNine uses necessary storage and optional analytics, session replay, and marketing services.",
+    path: "/cookies",
+  }),
+  robots: { index: false, follow: true },
+};
 
 export default function CookiesPage() {
   return (

app/faq/page.tsx

@@ -4,11 +4,14 @@ import { faqGroups } from "@/lib/faq";
 import { serializeJsonLd } from "@/lib/json-ld";
 import { createPublicPageMetadata } from "@/lib/seo";
 
-export const metadata = createPublicPageMetadata({
-  title: "AI agent directory FAQ: setup and verification",
-  description: "Answers about AI-agent listings, setup guides, source details, recorded checks, and what AgentNine verification marks mean.",
-  path: "/faq",
-});
+export const metadata = {
+  ...createPublicPageMetadata({
+    title: "AI agent directory FAQ: setup and verification",
+    description: "Answers about AI-agent listings, setup guides, source details, recorded checks, and what AgentNine verification marks mean.",
+    path: "/faq",
+  }),
+  robots: { index: false, follow: true },
+};
 
 export default function FaqPage() {
   const jsonLd = {

app/join/page.tsx

@@ -2,11 +2,14 @@ import type { Metadata } from "next";
 import { JoinForm } from "@/components/JoinForm";
 import { createPublicPageMetadata } from "@/lib/seo";
 
-export const metadata: Metadata = createPublicPageMetadata({
-  title: "Join AgentNine",
-  description: "Propose a contribution to AgentNine’s AI agent directory.",
-  path: "/join",
-});
+export const metadata: Metadata = {
+  ...createPublicPageMetadata({
+    title: "Join AgentNine",
+    description: "Propose a contribution to AgentNine’s AI agent directory.",
+    path: "/join",
+  }),
+  robots: { index: false, follow: true },
+};
 
 export default function JoinPage() {
   return (

app/login/page.tsx

@@ -1,7 +1,7 @@
 import { AuthForm } from "@/components/AuthForm";
 import { AuthPageShell } from "@/components/AuthPageShell";
 
-export const metadata = { title: "Sign in", robots: { index: false, follow: false } };
+export const metadata = { title: "Sign in", robots: { index: false, follow: true } };
 
 export default async function LoginPage({ searchParams }: { searchParams: Promise<{ authError?: string }> }) {
   const { authError } = await searchParams;

app/methodology/page.tsx

@@ -3,11 +3,14 @@ import Link from "next/link";
 import ArrowRight from "reicon-react/icons/ArrowRight";
 import { createPublicPageMetadata } from "@/lib/seo";
 
-export const metadata: Metadata = createPublicPageMetadata({
-  title: "Trust and methodology",
-  description: "How AgentNine records verification evidence, freshness, and uncertainty.",
-  path: "/methodology",
-});
+export const metadata: Metadata = {
+  ...createPublicPageMetadata({
+    title: "Trust and methodology",
+    description: "How AgentNine records verification evidence, freshness, and uncertainty.",
+    path: "/methodology",
+  }),
+  robots: { index: false, follow: true },
+};
 
 export default function MethodologyPage() {
   return (

app/news/[slug]/page.tsx

@@ -25,6 +25,7 @@ export async function generateMetadata({ params }: { params: Promise<{ slug: str
     description: post.summary,
     path: `/news/${post.slug}`,
     type: "article",
+    robots: { index: true, follow: true },
   });
 }
 

app/privacy/page.tsx

@@ -2,11 +2,14 @@ import Link from "next/link";
 import { createPublicPageMetadata } from "@/lib/seo";
 import { legalConfig } from "@/lib/legal-config";
 
-export const metadata = createPublicPageMetadata({
-  title: "Privacy policy",
-  description: "How AgentNine handles account, contact, feedback, preference, analytics, and session replay information.",
-  path: "/privacy",
-});
+export const metadata = {
+  ...createPublicPageMetadata({
+    title: "Privacy policy",
+    description: "How AgentNine handles account, contact, feedback, preference, analytics, and session replay information.",
+    path: "/privacy",
+  }),
+  robots: { index: false, follow: true },
+};
 
 const processorRows = [
   { provider: "Supabase", purpose: "Database, authentication, preferences, feedback, and account data" },

app/profile/page.tsx

@@ -1,7 +1,7 @@
 import { ProfileDetails } from "@/components/ProfileDetails";
 
 export const dynamic = "force-dynamic";
-export const metadata = { title: "Your profile", robots: { index: false, follow: false } };
+export const metadata = { title: "Your profile", robots: { index: false, follow: true } };
 
 export default function ProfilePage() {
   return (

app/signup/page.tsx

@@ -1,7 +1,7 @@
 import { AuthForm } from "@/components/AuthForm";
 import { AuthPageShell } from "@/components/AuthPageShell";
 
-export const metadata = { title: "Create an account", robots: { index: false, follow: false } };
+export const metadata = { title: "Create an account", robots: { index: false, follow: true } };
 
 export default async function SignupPage({ searchParams }: { searchParams: Promise<{ authError?: string }> }) {
   const { authError } = await searchParams;

app/sitemap.ts

@@ -4,10 +4,8 @@ import { getPublishedNewsPosts } from "@/lib/news";
 import {
   getLatestSitemapLastModified,
   getSitemapLastModified,
-  STATIC_PAGE_LAST_MODIFIED,
 } from "@/lib/sitemap";
 import { getLatestPublishedSiteFeedbackUpdate } from "@/lib/site-feedback-data";
-import { legalConfig } from "@/lib/legal-config";
 import { SITE_URL } from "@/lib/site";
 
 export const dynamic = "force-dynamic";
@@ -45,28 +43,11 @@ export default async function sitemap(): Promise<MetadataRoute.Sitemap> {
   const latestNewsUpdate = getLatestSitemapLastModified(
     newsPosts.map((post) => post.updated_at ?? post.published_at ?? post.created_at),
   );
-  const legalLastModified = getSitemapLastModified(legalConfig.lastUpdated);
-  const staticPageLastModified = {
-    about: getSitemapLastModified(STATIC_PAGE_LAST_MODIFIED.about),
-    faq: getSitemapLastModified(STATIC_PAGE_LAST_MODIFIED.faq),
-    join: getSitemapLastModified(STATIC_PAGE_LAST_MODIFIED.join),
-    contact: getSitemapLastModified(STATIC_PAGE_LAST_MODIFIED.contact),
-    methodology: getSitemapLastModified(STATIC_PAGE_LAST_MODIFIED.methodology),
-  };
-
   return [
     { url: base, ...(latestHomeUpdate ? { lastModified: latestHomeUpdate } : {}), changeFrequency: "weekly", priority: 1 },
     { url: `${base}/agents`, ...(latestAgentUpdate ? { lastModified: latestAgentUpdate } : {}), changeFrequency: "weekly", priority: 0.8 },
     { url: `${base}/news`, ...(latestNewsUpdate ? { lastModified: latestNewsUpdate } : {}), changeFrequency: "daily", priority: 0.7 },
     { url: `${base}/categories`, ...(latestAgentUpdate ? { lastModified: latestAgentUpdate } : {}), changeFrequency: "weekly", priority: 0.8 },
-    { url: `${base}/methodology`, ...(staticPageLastModified.methodology ? { lastModified: staticPageLastModified.methodology } : {}), changeFrequency: "monthly", priority: 0.5 },
-    { url: `${base}/join`, ...(staticPageLastModified.join ? { lastModified: staticPageLastModified.join } : {}), changeFrequency: "monthly", priority: 0.4 },
-    { url: `${base}/about`, ...(staticPageLastModified.about ? { lastModified: staticPageLastModified.about } : {}), changeFrequency: "monthly", priority: 0.5 },
-    { url: `${base}/faq`, ...(staticPageLastModified.faq ? { lastModified: staticPageLastModified.faq } : {}), changeFrequency: "monthly", priority: 0.5 },
-    { url: `${base}/contact`, ...(staticPageLastModified.contact ? { lastModified: staticPageLastModified.contact } : {}), changeFrequency: "monthly", priority: 0.3 },
-    { url: `${base}/privacy`, ...(legalLastModified ? { lastModified: legalLastModified } : {}), changeFrequency: "yearly", priority: 0.2 },
-    { url: `${base}/cookies`, ...(legalLastModified ? { lastModified: legalLastModified } : {}), changeFrequency: "yearly", priority: 0.2 },
-    { url: `${base}/terms`, ...(legalLastModified ? { lastModified: legalLastModified } : {}), changeFrequency: "yearly", priority: 0.2 },
     ...categories.map((category) => {
       const categoryLastModified = getLatestSitemapLastModified(
         agents.filter((agent) => agent.category_id === category.id).map((agent) => agent.updated_at),

app/terms/page.tsx

@@ -2,11 +2,14 @@ import Link from "next/link";
 import { createPublicPageMetadata } from "@/lib/seo";
 import { legalConfig } from "@/lib/legal-config";
 
-export const metadata = createPublicPageMetadata({
-  title: "Terms of service",
-  description: "Terms for using AgentNine and reviewing listed AI-agent projects.",
-  path: "/terms",
-});
+export const metadata = {
+  ...createPublicPageMetadata({
+    title: "Terms of service",
+    description: "Terms for using AgentNine and reviewing listed AI-agent projects.",
+    path: "/terms",
+  }),
+  robots: { index: false, follow: true },
+};
 
 export default function TermsPage() {
   const contact = legalConfig.contactEmail

lib/seo.ts

@@ -6,6 +6,7 @@ type PublicPageMetadata = {
   description: string;
   path: string;
   type?: "website" | "article";
+  robots?: Metadata["robots"];
 };
 
 const DEFAULT_SOCIAL_IMAGE = "/og-image.png";
@@ -44,6 +45,7 @@ export function createPublicPageMetadata({
   description,
   path,
   type = "website",
+  robots,
 }: PublicPageMetadata): Metadata {
   const canonicalUrl = new URL(path, SITE_URL).toString();
   const metaDescription = normalizeDescription(description);
@@ -58,6 +60,7 @@ export function createPublicPageMetadata({
   return {
     title,
     description: metaDescription,
+    ...(robots ? { robots } : {}),
     alternates: { canonical: canonicalUrl },
     openGraph: {
       title: socialTitle,

lib/sitemap.ts

@@ -1,11 +1,3 @@
-export const STATIC_PAGE_LAST_MODIFIED = {
-  about: "2026-10-19",
-  faq: "2026-10-17",
-  join: "2026-10-09",
-  contact: "2026-10-12",
-  methodology: "2026-10-17",
-} as const;
-
 export function getSitemapLastModified(value: string | null | undefined): Date | undefined {
   if (!value) return undefined;
   const timestamp = new Date(value).getTime();

tests/sitemap-generation.test.ts

@@ -63,25 +63,30 @@ describe("sitemap generation", () => {
     expect(entries.find((entry) => entry.url.endsWith("/news/second-story"))?.lastModified).toEqual(lastAgentUpdate);
   });
 
-  it("includes last-modified dates for legal pages from the legal revision date", async () => {
+  it("excludes pages marked noindex from the sitemap", async () => {
     const entries = await sitemap();
-    const expected = new Date("October 19, 2026");
-    for (const path of ["/privacy", "/cookies", "/terms"]) {
-      expect(entries.find((entry) => entry.url.endsWith(path))?.lastModified).toEqual(expected);
+    for (const path of [
+      "/about",
+      "/methodology",
+      "/join",
+      "/faq",
+      "/contact",
+      "/privacy",
+      "/cookies",
+      "/terms",
+    ]) {
+      expect(entries.find((entry) => entry.url === `https://agentnine.pro${path}`)).toBeUndefined();
     }
   });
 
-  it("includes the last content edit date for the remaining static pages", async () => {
+  it("keeps published agents, categories, and news articles in the sitemap", async () => {
     const entries = await sitemap();
-    const expectedDates = {
-      "/about": "2026-10-19",
-      "/faq": "2026-10-17",
-      "/join": "2026-10-09",
-      "/contact": "2026-10-12",
-      "/methodology": "2026-10-17",
-    };
-    for (const [path, date] of Object.entries(expectedDates)) {
-      expect(entries.find((entry) => entry.url.endsWith(path))?.lastModified).toEqual(new Date(date));
+    for (const path of [
+      "/agents/first-agent",
+      "/categories/first-category",
+      "/news/first-story",
+    ]) {
+      expect(entries.find((entry) => entry.url === `https://agentnine.pro${path}`)).toBeDefined();
     }
   });
 });

tests/static-indexing-policy.test.ts

@@ -0,0 +1,42 @@
+import { describe, expect, it } from "vitest";
+import { metadata as aboutMetadata } from "@/app/about/page";
+import { metadata as methodologyMetadata } from "@/app/methodology/page";
+import { metadata as joinMetadata } from "@/app/join/page";
+import { metadata as faqMetadata } from "@/app/faq/page";
+import { metadata as contactMetadata } from "@/app/contact/layout";
+import { metadata as privacyMetadata } from "@/app/privacy/page";
+import { metadata as cookiesMetadata } from "@/app/cookies/page";
+import { metadata as termsMetadata } from "@/app/terms/page";
+import { metadata as feedbackMetadata } from "@/app/feedback/page";
+import { metadata as privacyChoicesMetadata } from "@/app/privacy/choices/page";
+import { metadata as loginMetadata } from "@/app/login/page";
+import { metadata as signupMetadata } from "@/app/signup/page";
+import { metadata as profileMetadata } from "@/app/profile/page";
+import { metadata as accountMetadata } from "@/app/account/page";
+
+describe("static pages excluded from search indexing", () => {
+  it.each([
+    ["/about", aboutMetadata],
+    ["/methodology", methodologyMetadata],
+    ["/join", joinMetadata],
+    ["/faq", faqMetadata],
+    ["/contact", contactMetadata],
+    ["/privacy", privacyMetadata],
+    ["/cookies", cookiesMetadata],
+    ["/terms", termsMetadata],
+  ])("keeps %s crawlable but noindex", (_path, metadata) => {
+    expect(metadata.robots).toEqual({ index: false, follow: true });
+    expect(metadata.alternates?.canonical).toBeTruthy();
+  });
+
+  it.each([
+    ["/feedback", feedbackMetadata],
+    ["/privacy/choices", privacyChoicesMetadata],
+    ["/login", loginMetadata],
+    ["/signup", signupMetadata],
+    ["/profile", profileMetadata],
+    ["/account", accountMetadata],
+  ])("keeps %s crawlable while excluding it from search", (_path, metadata) => {
+    expect(metadata.robots).toEqual({ index: false, follow: true });
+  });
+});