# AI-Friendly robots.txt
# Allows AI crawlers to index content for training and search

# Search Engine Crawlers (Standard)
User-agent: Googlebot
Disallow:

User-agent: Googlebot-Image
Disallow:

User-agent: Googlebot-News
Disallow:

User-agent: Bingbot
Disallow:

User-agent: MSNBot
Disallow:

User-agent: Slurp
Disallow:

User-agent: DuckDuckBot
Disallow:

User-agent: Baiduspider
Disallow:

User-agent: YandexBot
Disallow:

User-agent: Applebot
Disallow:

# AI Crawlers - ALLOWED
User-agent: ClaudeBot
Disallow:

User-agent: Claude-SearchBot
Disallow:

User-agent: GPTBot
Disallow:

User-agent: ChatGPT-User
Disallow:

User-agent: OAI-SearchBot
Disallow:

User-agent: Google-Extended
Disallow:

User-agent: PerplexityBot
Disallow:

User-agent: anthropic-ai
Disallow:

User-agent: Amazonbot
Disallow:

User-agent: FacebookBot
Disallow:

User-agent: Twitterbot
Disallow:

User-agent: LinkedInBot
Disallow:

# SEO/Analytics Crawlers
User-agent: AhrefsBot
Disallow:

User-agent: SemrushBot
Disallow:

User-agent: MJ12bot
Disallow:

User-agent: DotBot
Disallow:

User-agent: rogerbot
Disallow:

User-agent: Screaming Frog SEO Spider
Disallow:

# Archive/Research
User-agent: ia_archiver
Disallow:

User-agent: archive.org_bot
Disallow:

# Sitemap location
Sitemap: /sitemap.xml