﻿# Robots.txt for www.odoo-expertos.com
# Last updated: 2026-01-26
# Optimized for SEO: Google, Bing, and LLM crawlers
# Total pages: 1,210 across 6 languages (ES, EN, DE, FR, PT, AR)

# ============================================
# SITEMAP INDEX (Points to all 6 language sitemaps)
# ============================================
Sitemap: https://www.odoo-expertos.com/sitemap.xml

# ============================================
# LLMs.txt FOR AI CRAWLERS (ChatGPT, Perplexity, Claude)
# ============================================
# Brief summary: https://www.odoo-expertos.com/llms.txt
# Full index: https://www.odoo-expertos.com/llms-full.txt

# ============================================
# ALL CRAWLERS - DEFAULT RULES
# ============================================
User-agent: *
Allow: /
Disallow: /api/
Disallow: /components/
Disallow: /screenshots/
Disallow: /.git/
Disallow: /node_modules/
Disallow: /config/
Disallow: /addons/
Disallow: /.env
Disallow: /.claude/
Disallow: /docker-compose*.yml
Disallow: /Dockerfile*
Disallow: /*-test.html
Disallow: /*-backup.html
Disallow: /test-*.html
Disallow: /temp-*.html
Disallow: /*.md$
Disallow: /*.sh$
Disallow: /*.conf$
Crawl-delay: 1

# ============================================
# GOOGLE SPECIFIC RULES
# ============================================
User-agent: Googlebot
Allow: /
Allow: /css/
Allow: /js/
Allow: /odoo/
Allow: /odoo-hosting/
Allow: /odoo-ia/
Allow: /en/
Allow: /de/
Allow: /fr/
Allow: /pt/
Allow: /ar/
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /api/
Disallow: /components/
Disallow: /.git/
Crawl-delay: 0

User-agent: Googlebot-Image
Allow: /
Allow: /*.jpg$
Allow: /*.jpeg$
Allow: /*.png$
Allow: /*.webp$
Allow: /*.svg$
Disallow: /screenshots/
Crawl-delay: 0

# ============================================
# BING SPECIFIC RULES  
# ============================================
User-agent: Bingbot
Allow: /
Allow: /css/
Allow: /js/
Allow: /odoo/
Allow: /odoo-hosting/
Allow: /odoo-ia/
Disallow: /api/
Disallow: /components/
Disallow: /.git/
Crawl-delay: 1

# ============================================
# LLM & AI CRAWLER SUPPORT (ALL KNOWN BOTS)
# ============================================
# We welcome AI crawlers to index our content
# LLMs.txt available at /llms.txt and /llms-full.txt

# --- OpenAI Bots ---
User-agent: GPTBot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /api/
Disallow: /components/
Crawl-delay: 1

User-agent: ChatGPT-User
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Crawl-delay: 1

User-agent: OAI-SearchBot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Crawl-delay: 1

# --- Anthropic Bots ---
User-agent: ClaudeBot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /api/
Disallow: /components/
Crawl-delay: 1

User-agent: Claude-Web
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Crawl-delay: 1

User-agent: anthropic-ai
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Crawl-delay: 1

# --- Google AI Bots ---
User-agent: Google-Extended
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /api/
Disallow: /components/
Crawl-delay: 1

User-agent: GoogleOther
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Crawl-delay: 1

# --- Perplexity Bots ---
User-agent: PerplexityBot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Crawl-delay: 1

# --- Microsoft/Bing AI Bots ---
User-agent: Bingbot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Crawl-delay: 1

User-agent: BingPreview
Allow: /
Crawl-delay: 1

# --- Meta AI Bots ---
User-agent: FacebookBot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Crawl-delay: 1

User-agent: meta-externalagent
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Crawl-delay: 1

User-agent: Meta-ExternalFetcher
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Crawl-delay: 1

# --- Apple AI Bots ---
User-agent: Applebot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Crawl-delay: 1

User-agent: Applebot-Extended
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Crawl-delay: 1

# --- Cohere AI ---
User-agent: cohere-ai
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Crawl-delay: 1

# --- Amazon/AWS AI ---
User-agent: Amazonbot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Crawl-delay: 1

# --- You.com AI ---
User-agent: YouBot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Crawl-delay: 1

# --- Hugging Face ---
User-agent: HuggingFaceBot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Crawl-delay: 2

# --- AI Search Engines ---
User-agent: AI2Bot
Allow: /
Crawl-delay: 2

User-agent: Diffbot
Allow: /
Crawl-delay: 2

User-agent: omgili
Allow: /
Crawl-delay: 2

User-agent: omgilibot
Allow: /
Crawl-delay: 2

# --- Common Crawl (used for AI training) ---
User-agent: CCBot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Crawl-delay: 2

# --- Brave Search AI ---
User-agent: BraveBot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Crawl-delay: 1

# --- DuckDuckGo AI ---
User-agent: DuckDuckBot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Crawl-delay: 1

# --- Neeva AI (now part of Snowflake) ---
User-agent: NeevaBot
Allow: /
Crawl-delay: 2

# --- Writesonic/Chatsonic ---
User-agent: WritesonicBot
Allow: /
Crawl-delay: 2

# --- Jasper AI ---
User-agent: JasperBot
Allow: /
Crawl-delay: 2

# --- Copy.ai ---
User-agent: CopyaiBot
Allow: /
Crawl-delay: 2

# --- Other AI Research Crawlers ---
User-agent: DataForSeoBot
Allow: /
Crawl-delay: 2

User-agent: webz.io
Allow: /
Crawl-delay: 2

User-agent: iaskspider
Allow: /
Crawl-delay: 2

User-agent: Scrapy
Allow: /
Crawl-delay: 3

# Common LLM crawlers
User-agent: CCBot
Allow: /
Crawl-delay: 2

# ============================================
# SOCIAL MEDIA CRAWLERS
# ============================================
User-agent: facebookexternalhit
Allow: /
Crawl-delay: 0

User-agent: Twitterbot
Allow: /
Crawl-delay: 0

User-agent: LinkedInBot
Allow: /
Crawl-delay: 1

# ============================================
# MONITORING & LEGITIMATE SERVICES
# ============================================
User-agent: UptimeRobot
Allow: /
Crawl-delay: 0

User-agent: Slackbot
Allow: /
Crawl-delay: 0

# ============================================
# BAD BOTS - BLOCK COMPLETELY
# ============================================
User-agent: AhrefsBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: PetalBot
Disallow: /

# ============================================
# PERFORMANCE & CRAWL BUDGET OPTIMIZATION
# ============================================
# Prevent crawling of duplicate content
User-agent: *
Disallow: /*?
Disallow: /*&
Disallow: /print/
Disallow: /mailto/
Disallow: /*#
