# AI answer-engine and LLM crawlers are welcome, so MCP Ads can be cited in
# AI-generated answers. They share the one rule group below on purpose: a
# crawler obeys only the most specific group that names it, so the separate
# "User-agent: GPTBot / Allow: /" groups this file used to have exempted every
# AI crawler from the Disallow rules and let them into /manage, /admin, /api/
# and the /mcp endpoint itself.
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: ClaudeBot
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: Claude-Web
User-agent: anthropic-ai
User-agent: CCBot
User-agent: Google-Extended
User-agent: GoogleOther
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Meta-ExternalAgent
User-agent: Meta-ExternalFetcher
User-agent: Applebot-Extended
User-agent: Bytespider
User-agent: xAI-Bot
User-agent: cohere-ai
User-agent: MistralAI-User
User-agent: *
# Disallow lines come first. Google applies the longest matching rule, so the
# order means nothing to it, but some crawlers take the first match, and for
# them a leading "Allow: /" used to cancel every rule below it.
# Disallow app pages (no marketing value, no indexing needed)
Disallow: /onboard
Disallow: /welcome
Disallow: /manage
Disallow: /console
# /connect only (the $ anchors it): /connections/* are 301s now and Googlebot
# must be able to fetch them to see where they went.
Disallow: /connect$
Disallow: /api/
Disallow: /auth/
Disallow: /billing/
Disallow: /user/
Disallow: /admin
Disallow: /mcp
Allow: /

Sitemap: https://mcp-ads.com/sitemap.xml
Sitemap: https://mcp-ads.com/docs-sitemap.xml

# A curated map for LLM crawlers/answer engines lives at /llms.txt. It
# deliberately excludes the generated /docs/api-reference/{platform}/{tool}
# pages, which are gone (410) since 2026-09-24. See https://llmstxt.org/
# https://mcp-ads.com/llms.txt