137 lines
1.6 KiB
Plaintext
137 lines
1.6 KiB
Plaintext
# Allow all legitimate crawlers by default
|
|
User-agent: *
|
|
Allow: /
|
|
|
|
# Major Search Engines
|
|
User-agent: Googlebot
|
|
Allow: /
|
|
|
|
User-agent: Bingbot
|
|
Allow: /
|
|
|
|
User-agent: Slurp
|
|
Allow: /
|
|
|
|
User-agent: DuckDuckBot
|
|
Allow: /
|
|
|
|
User-agent: Baiduspider
|
|
Allow: /
|
|
|
|
User-agent: YandexBot
|
|
Allow: /
|
|
|
|
# AI and LLM Crawlers
|
|
User-agent: GPTBot
|
|
Allow: /
|
|
|
|
User-agent: ChatGPT-User
|
|
Allow: /
|
|
|
|
User-agent: CCBot
|
|
Allow: /
|
|
|
|
User-agent: anthropic-ai
|
|
Allow: /
|
|
|
|
User-agent: Claude-Web
|
|
Allow: /
|
|
|
|
User-agent: PerplexityBot
|
|
Allow: /
|
|
|
|
User-agent: YouBot
|
|
Allow: /
|
|
|
|
User-agent: Bytespider
|
|
Allow: /
|
|
|
|
# Social Media Bots
|
|
User-agent: facebookexternalhit
|
|
Allow: /
|
|
|
|
User-agent: LinkedInBot
|
|
Allow: /
|
|
|
|
User-agent: WhatsApp
|
|
Allow: /
|
|
|
|
User-agent: TwitterBot
|
|
Allow: /
|
|
|
|
User-agent: TelegramBot
|
|
Allow: /
|
|
|
|
User-agent: Discordbot
|
|
Allow: /
|
|
|
|
User-agent: Slackbot
|
|
Allow: /
|
|
|
|
# SEO and Analysis Tools
|
|
User-agent: AhrefsBot
|
|
Allow: /
|
|
|
|
User-agent: SemrushBot
|
|
Allow: /
|
|
|
|
User-agent: MJ12bot
|
|
Allow: /
|
|
|
|
User-agent: DotBot
|
|
Allow: /
|
|
|
|
User-agent: DataForSeoBot
|
|
Allow: /
|
|
|
|
# Archive and Research
|
|
User-agent: ia_archiver
|
|
Allow: /
|
|
|
|
User-agent: archive.org_bot
|
|
Allow: /
|
|
|
|
User-agent: Wayback Machine
|
|
Allow: /
|
|
|
|
# Apple Services
|
|
User-agent: Applebot
|
|
Allow: /
|
|
|
|
User-agent: AppleNewsBot
|
|
Allow: /
|
|
|
|
# Amazon
|
|
User-agent: Amazonbot
|
|
Allow: /
|
|
|
|
# Microsoft
|
|
User-agent: msnbot
|
|
Allow: /
|
|
|
|
User-agent: bingbot
|
|
Allow: /
|
|
|
|
# Other Search Engines
|
|
User-agent: Qwantify
|
|
Allow: /
|
|
|
|
User-agent: Seznam
|
|
Allow: /
|
|
|
|
User-agent: Sogou
|
|
Allow: /
|
|
|
|
User-agent: Exabot
|
|
Allow: /
|
|
|
|
# Development and Testing
|
|
User-agent: Screaming Frog SEO Spider
|
|
Allow: /
|
|
|
|
# Crawl-delay for bots that respect it (in seconds)
|
|
User-agent: *
|
|
Crawl-delay: 1
|
|
|
|
# Sitemap
|
|
Sitemap: https://www.odoo-expertos.com/sitemap.xml |