# Xenition — https://xenition.com
#
# Content Signals (contentsignals.org): search=index in AI/search results, ai-input=use in AI answers (RAG), ai-train=train models. All yes = maximum discoverability.
#
# AI assistants are named explicitly below rather than left to the wildcard. Two reasons: several of
# these crawlers check for their own user-agent before deciding, and naming them is an unambiguous
# signal that we want to be read and cited. /app, /admin and /auth stay closed to everyone — that is
# the signed-in product, not content.

User-agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /app
Disallow: /admin
Disallow: /auth/
Disallow: /reset-password

# --- AI assistants and answer engines -----------------------------------------------------------
# OpenAI runs three: GPTBot trains, OAI-SearchBot builds the ChatGPT search index, and ChatGPT-User
# fetches a page live when someone asks about us right now. The third is the one that answers a
# prospect mid-conversation, so all three are welcome.
User-agent: GPTBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /

User-agent: OAI-SearchBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /

User-agent: ChatGPT-User
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /

# Anthropic
User-agent: ClaudeBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /

User-agent: Claude-Web
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /

User-agent: anthropic-ai
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /

# Perplexity
User-agent: PerplexityBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /

User-agent: Perplexity-User
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /

# Google's AI products honour Google-Extended separately from Googlebot's ordinary crawl.
User-agent: Google-Extended
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /

# Microsoft Copilot / Bing
User-agent: bingbot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /

# Apple Intelligence and Siri
User-agent: Applebot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /

User-agent: Applebot-Extended
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /

# Meta AI
User-agent: meta-externalagent
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /

# Common Crawl — the corpus a large number of downstream models train on.
User-agent: CCBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /

# Amazon
User-agent: Amazonbot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /

# ByteDance
User-agent: Bytespider
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /

# Mistral
User-agent: MistralAI-User
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /

# One line, because /sitemap.xml is a sitemap INDEX and lists the rest: the
# hand-written marketing pages, and the generated ones for tools, forms,
# templates, connectors, blog posts and marketplace listings.
#
# It used to list all seven here. Both forms are valid, but the structure of the
# site belongs in a sitemap rather than in a crawler directive file — and an
# index is what makes Search Console report coverage per child, which is the
# only practical way to see that one catalog is indexing while another is not.
Sitemap: https://xenition.com/sitemap.xml