# Robots.txt for www.odoo-expertos.com # Last updated: 2026-01-26 # Optimized for SEO: Google, Bing, and LLM crawlers # Total pages: 1,210 across 6 languages (ES, EN, DE, FR, PT, AR) # ============================================ # SITEMAP INDEX (Points to all 6 language sitemaps) # ============================================ Sitemap: https://www.odoo-expertos.com/sitemap.xml # ============================================ # LLMs.txt FOR AI CRAWLERS (ChatGPT, Perplexity, Claude) # ============================================ # Brief summary: https://www.odoo-expertos.com/llms.txt # Full index: https://www.odoo-expertos.com/llms-full.txt # ============================================ # ALL CRAWLERS - DEFAULT RULES # ============================================ User-agent: * Allow: / Disallow: /api/ Disallow: /components/ Disallow: /screenshots/ Disallow: /.git/ Disallow: /node_modules/ Disallow: /config/ Disallow: /addons/ Disallow: /.env Disallow: /.claude/ Disallow: /docker-compose*.yml Disallow: /Dockerfile* Disallow: /*-test.html Disallow: /*-backup.html Disallow: /test-*.html Disallow: /temp-*.html Disallow: /*.md$ Disallow: /*.sh$ Disallow: /*.conf$ Disallow: /*? Disallow: /*& Disallow: /print/ Disallow: /mailto/ Crawl-delay: 1 # ============================================ # GOOGLE SPECIFIC RULES # ============================================ User-agent: Googlebot Allow: / Allow: /css/ Allow: /js/ Allow: /odoo/ Allow: /odoo-hosting/ Allow: /odoo-ia/ Allow: /en/ Allow: /de/ Allow: /fr/ Allow: /pt/ Allow: /ar/ Allow: /llms.txt Allow: /llms-full.txt Disallow: /api/ Disallow: /components/ Disallow: /.git/ Crawl-delay: 0 User-agent: Googlebot-Image Allow: / Allow: /*.jpg$ Allow: /*.jpeg$ Allow: /*.png$ Allow: /*.webp$ Allow: /*.svg$ Disallow: /screenshots/ Crawl-delay: 0 # ============================================ # BING SPECIFIC RULES # ============================================ User-agent: Bingbot Allow: / Allow: /css/ Allow: /js/ Allow: /odoo/ Allow: /odoo-hosting/ Allow: /odoo-ia/ Disallow: /api/ Disallow: /components/ Disallow: /.git/ Crawl-delay: 1 # ============================================ # LLM & AI CRAWLER SUPPORT (ALL KNOWN BOTS) # ============================================ # We welcome AI crawlers to index our content # LLMs.txt available at /llms.txt and /llms-full.txt # --- OpenAI Bots --- User-agent: GPTBot Allow: / Allow: /llms.txt Allow: /llms-full.txt Disallow: /api/ Disallow: /components/ Crawl-delay: 1 User-agent: ChatGPT-User Allow: / Allow: /llms.txt Allow: /llms-full.txt Crawl-delay: 1 User-agent: OAI-SearchBot Allow: / Allow: /llms.txt Allow: /llms-full.txt Crawl-delay: 1 # --- Anthropic Bots --- User-agent: ClaudeBot Allow: / Allow: /llms.txt Allow: /llms-full.txt Disallow: /api/ Disallow: /components/ Crawl-delay: 1 User-agent: Claude-Web Allow: / Allow: /llms.txt Allow: /llms-full.txt Crawl-delay: 1 User-agent: anthropic-ai Allow: / Allow: /llms.txt Allow: /llms-full.txt Crawl-delay: 1 # --- Google AI Bots --- User-agent: Google-Extended Allow: / Allow: /llms.txt Allow: /llms-full.txt Disallow: /api/ Disallow: /components/ Crawl-delay: 1 User-agent: GoogleOther Allow: / Allow: /llms.txt Allow: /llms-full.txt Crawl-delay: 1 # --- Perplexity Bots --- User-agent: PerplexityBot Allow: / Allow: /llms.txt Allow: /llms-full.txt Crawl-delay: 1 # --- Microsoft/Bing AI Bots --- User-agent: BingPreview Allow: / Crawl-delay: 1 # --- Meta AI Bots --- User-agent: FacebookBot Allow: / Allow: /llms.txt Allow: /llms-full.txt Crawl-delay: 1 User-agent: meta-externalagent Allow: / Allow: /llms.txt Allow: /llms-full.txt Crawl-delay: 1 User-agent: Meta-ExternalFetcher Allow: / Allow: /llms.txt Allow: /llms-full.txt Crawl-delay: 1 # --- Apple AI Bots --- User-agent: Applebot Allow: / Allow: /llms.txt Allow: /llms-full.txt Crawl-delay: 1 User-agent: Applebot-Extended Allow: / Allow: /llms.txt Allow: /llms-full.txt Crawl-delay: 1 # --- Cohere AI --- User-agent: cohere-ai Allow: / Allow: /llms.txt Allow: /llms-full.txt Crawl-delay: 1 # --- Amazon/AWS AI --- User-agent: Amazonbot Allow: / Allow: /llms.txt Allow: /llms-full.txt Crawl-delay: 1 # --- You.com AI --- User-agent: YouBot Allow: / Allow: /llms.txt Allow: /llms-full.txt Crawl-delay: 1 # --- Hugging Face --- User-agent: HuggingFaceBot Allow: / Allow: /llms.txt Allow: /llms-full.txt Crawl-delay: 2 # --- AI Search Engines --- User-agent: AI2Bot Allow: / Crawl-delay: 2 User-agent: Diffbot Allow: / Crawl-delay: 2 User-agent: omgili Allow: / Crawl-delay: 2 User-agent: omgilibot Allow: / Crawl-delay: 2 # --- Common Crawl (used for AI training) --- User-agent: CCBot Allow: / Allow: /llms.txt Allow: /llms-full.txt Crawl-delay: 2 # --- Brave Search AI --- User-agent: BraveBot Allow: / Allow: /llms.txt Allow: /llms-full.txt Crawl-delay: 1 # --- DuckDuckGo AI --- User-agent: DuckDuckBot Allow: / Allow: /llms.txt Allow: /llms-full.txt Crawl-delay: 1 # --- Neeva AI (now part of Snowflake) --- User-agent: NeevaBot Allow: / Crawl-delay: 2 # --- Writesonic/Chatsonic --- User-agent: WritesonicBot Allow: / Crawl-delay: 2 # --- Jasper AI --- User-agent: JasperBot Allow: / Crawl-delay: 2 # --- Copy.ai --- User-agent: CopyaiBot Allow: / Crawl-delay: 2 # --- Other AI Research Crawlers --- User-agent: DataForSeoBot Allow: / Crawl-delay: 2 User-agent: webz.io Allow: / Crawl-delay: 2 User-agent: iaskspider Allow: / Crawl-delay: 2 User-agent: Scrapy Allow: / Crawl-delay: 3 # ============================================ # SOCIAL MEDIA CRAWLERS # ============================================ User-agent: facebookexternalhit Allow: / Crawl-delay: 0 User-agent: Twitterbot Allow: / Crawl-delay: 0 User-agent: LinkedInBot Allow: / Crawl-delay: 1 # ============================================ # MONITORING & LEGITIMATE SERVICES # ============================================ User-agent: UptimeRobot Allow: / Crawl-delay: 0 User-agent: Slackbot Allow: / Crawl-delay: 0 # ============================================ # BAD BOTS - BLOCK COMPLETELY # ============================================ User-agent: AhrefsBot Disallow: / User-agent: MJ12bot Disallow: / User-agent: DotBot Disallow: / User-agent: SemrushBot Disallow: / User-agent: Bytespider Disallow: / User-agent: PetalBot Disallow: /