robots.txt Generator for AI Crawlers

# robots.txt - generated by BrowserLeaksUS /ai/robots-ai/

User-agent: *
Disallow:

# GPTBot - OpenAI (Crawls content to train GPT models.)
User-agent: GPTBot
Disallow: /

# ChatGPT-User - OpenAI (Fetches pages on-demand when ChatGPT users click links.)
User-agent: ChatGPT-User
Disallow:

# OAI-SearchBot - OpenAI (Indexes pages for ChatGPT Search.)
User-agent: OAI-SearchBot
Disallow:

# ClaudeBot - Anthropic (Crawls content for Claude training and retrieval.)
User-agent: ClaudeBot
Disallow:

# anthropic-ai - Anthropic (Anthropic training crawler (legacy name).)
User-agent: anthropic-ai
Disallow: /

# Claude-Web - Anthropic (On-demand fetcher for Claude conversations.)
User-agent: Claude-Web
Disallow:

# PerplexityBot - Perplexity (Indexes pages for Perplexity search.)
User-agent: PerplexityBot
Disallow:

# Perplexity-User - Perplexity (Fetches pages when Perplexity users click links.)
User-agent: Perplexity-User
Disallow:

# Google-Extended - Google (robots.txt-only token - controls use for Gemini / Vertex AI (does NOT affect Google Search).)
User-agent: Google-Extended
Disallow: /

# Applebot-Extended - Apple (robots.txt-only token - controls use for Apple AI model training.)
User-agent: Applebot-Extended
Disallow: /

# CCBot - Common Crawl (Common Crawl archive - widely used in LLM training sets.)
User-agent: CCBot
Disallow: /

# Bytespider - ByteDance (ByteDance / TikTok AI training crawler.)
User-agent: Bytespider
Disallow: /

# Meta-ExternalAgent - Meta (Meta / Facebook AI training crawler.)
User-agent: Meta-ExternalAgent
Disallow: /

# Meta-ExternalFetcher - Meta (Meta on-demand fetcher.)
User-agent: Meta-ExternalFetcher
Disallow:

# Amazonbot - Amazon (Amazon / Alexa AI training crawler.)
User-agent: Amazonbot
Disallow: /

# YouBot - You.com (You.com search crawler.)
User-agent: YouBot
Disallow:

# cohere-ai - Cohere (Cohere training crawler.)
User-agent: cohere-ai
Disallow: /

# DuckAssistBot - DuckDuckGo (DuckDuckGo AI assistant fetcher.)
User-agent: DuckAssistBot
Disallow:

# MistralAI-User - Mistral (Mistral on-demand fetcher.)
User-agent: MistralAI-User
Disallow:

# FacebookBot - Meta (Facebook AI / translation crawler.)
User-agent: FacebookBot
Disallow: /

# Diffbot - Diffbot (Diffbot web extraction / AI training.)
User-agent: Diffbot
Disallow: /

# Omgilibot - Webz.io (Webz.io / Omgili data collection for AI.)
User-agent: Omgilibot
Disallow: /

# Timpibot - Timpi (Timpi decentralized search / AI.)
User-agent: Timpibot
Disallow: /