# robots.txt - mostpopularaitools.com
# Stack: Next.js 16 + Supabase + Cloudflare

User-agent: *

# Allow render assets
Allow: /_next/static/
Allow: /_next/data/

# Block private/internal routes
Disallow: /api/
Disallow: /admin/
Disallow: /auth/
Disallow: /go/
Disallow: /_next/cache/
Disallow: /cdn-cgi/

# ---------------------------------------------------------------------------
# NOTE ON QUERY PARAMETERS
#
# This file used to Disallow /*?sort=, /*?search=, /*?ref= and /*?utm_*.
# Those rules were removed deliberately.
#
# Blocking a URL and noindexing it are mutually exclusive: a blocked URL is
# never fetched, so the noindex is never read, and Google may still index the
# bare URL ("Indexed, though blocked by robots.txt"). The /reviews search and
# order links (/reviews?search=..., /reviews?sort=name) are internal links, so
# they must stay crawlable for their noindex to be seen.
#
# Handling now lives where it belongs, server-side:
#   - lib/listing-url-seo.ts   -> emits `noindex, follow` for ?search= and
#                                 ?sort=
#   - middleware.ts            -> 301-strips utm_*, ref and dpl (gclid and fbclid
#                                 are left for ad attribution; the canonical drops
#                                 them)
#   - generateMetadata()       -> canonical points at the clean route URL
#
# Google can now crawl these, read the noindex, drop them from the index, and
# still follow the links through to the reviews.
#
# NOTE ON REMOVED ROUTES (Stage 1K-RESET, 2026-09-22)
#
# The Disallow rules for /dashboard/, /checkout/ and /r/ were removed with the
# routes themselves. A rule for a path that no longer exists protects nothing,
# and a 404 needs no robots rule. /dashboard came back on 2026-09-25 as a
# signed-in page; like /submit it redirects a signed-out visitor to sign in and
# is noindex, so it needs no rule either.
# ---------------------------------------------------------------------------

# ---------------------------------------------------------------------------
# AI CRAWLERS
#
# Explicitly allowed. This is a review site whose value is being cited as a
# source in AI answers, so being readable by answer engines is the point. Stated
# explicitly rather than inherited from User-agent: * so the policy is a
# decision on the record, not an accident of omission.
# See also /llms.txt for a machine-readable index of the reviews.
#
# A named group replaces the `User-agent: *` rules for that crawler rather
# than adding to them, so each group repeats every private path: without
# that, these bots were free to crawl /go/ (tracked outbound redirects),
# /_next/cache/ and /cdn-cgi/.
# ---------------------------------------------------------------------------

User-agent: GPTBot
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /auth/
Disallow: /go/
Disallow: /_next/cache/
Disallow: /cdn-cgi/

User-agent: ChatGPT-User
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /auth/
Disallow: /go/
Disallow: /_next/cache/
Disallow: /cdn-cgi/

User-agent: OAI-SearchBot
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /auth/
Disallow: /go/
Disallow: /_next/cache/
Disallow: /cdn-cgi/

User-agent: ClaudeBot
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /auth/
Disallow: /go/
Disallow: /_next/cache/
Disallow: /cdn-cgi/

User-agent: Claude-Web
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /auth/
Disallow: /go/
Disallow: /_next/cache/
Disallow: /cdn-cgi/

User-agent: PerplexityBot
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /auth/
Disallow: /go/
Disallow: /_next/cache/
Disallow: /cdn-cgi/

User-agent: Google-Extended
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /auth/
Disallow: /go/
Disallow: /_next/cache/
Disallow: /cdn-cgi/

User-agent: Applebot-Extended
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /auth/
Disallow: /go/
Disallow: /_next/cache/
Disallow: /cdn-cgi/

# Bulk scrapers that return no referral traffic and no citation.
User-agent: CCBot
Disallow: /

User-agent: Bytespider
Disallow: /

Sitemap: https://mostpopularaitools.com/sitemap.xml