# Keeping the site out of AI training sets and AI answer engines.
# These are separate jobs and separate agents: one crawls to train, another
# fetches a page because a user asked a question right now. Drop the groups
# you do not need.

# --- training corpora ---
User-agent: GPTBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: meta-externalagent
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: cohere-ai
Disallow: /

# --- fetching on behalf of a user asking a question ---
User-agent: ChatGPT-User
Disallow: /

User-agent: Claude-User
Disallow: /

User-agent: PerplexityBot
Disallow: /

# Ordinary search crawling stays allowed.
User-agent: *
Disallow: