new site
This commit is contained in:
oliver
@@ -0,0 +1,366 @@
|
||||
# Robots.txt for www.odoo-expertos.com
|
||||
# Last updated: 2026-01-26
|
||||
# Optimized for SEO: Google, Bing, and LLM crawlers
|
||||
# Total pages: 1,210 across 6 languages (ES, EN, DE, FR, PT, AR)
|
||||
|
||||
# ============================================
|
||||
# SITEMAP INDEX (Points to all 6 language sitemaps)
|
||||
# ============================================
|
||||
Sitemap: https://www.odoo-expertos.com/sitemap.xml
|
||||
|
||||
# ============================================
|
||||
# LLMs.txt FOR AI CRAWLERS (ChatGPT, Perplexity, Claude)
|
||||
# ============================================
|
||||
# Brief summary: https://www.odoo-expertos.com/llms.txt
|
||||
# Full index: https://www.odoo-expertos.com/llms-full.txt
|
||||
|
||||
# ============================================
|
||||
# ALL CRAWLERS - DEFAULT RULES
|
||||
# ============================================
|
||||
User-agent: *
|
||||
Allow: /
|
||||
Disallow: /api/
|
||||
Disallow: /components/
|
||||
Disallow: /screenshots/
|
||||
Disallow: /.git/
|
||||
Disallow: /node_modules/
|
||||
Disallow: /config/
|
||||
Disallow: /addons/
|
||||
Disallow: /.env
|
||||
Disallow: /.claude/
|
||||
Disallow: /docker-compose*.yml
|
||||
Disallow: /Dockerfile*
|
||||
Disallow: /*-test.html
|
||||
Disallow: /*-backup.html
|
||||
Disallow: /test-*.html
|
||||
Disallow: /temp-*.html
|
||||
Disallow: /*.md$
|
||||
Disallow: /*.sh$
|
||||
Disallow: /*.conf$
|
||||
Crawl-delay: 1
|
||||
|
||||
# ============================================
|
||||
# GOOGLE SPECIFIC RULES
|
||||
# ============================================
|
||||
User-agent: Googlebot
|
||||
Allow: /
|
||||
Allow: /css/
|
||||
Allow: /js/
|
||||
Allow: /odoo/
|
||||
Allow: /odoo-hosting/
|
||||
Allow: /odoo-ia/
|
||||
Allow: /en/
|
||||
Allow: /de/
|
||||
Allow: /fr/
|
||||
Allow: /pt/
|
||||
Allow: /ar/
|
||||
Allow: /llms.txt
|
||||
Allow: /llms-full.txt
|
||||
Disallow: /api/
|
||||
Disallow: /components/
|
||||
Disallow: /.git/
|
||||
Crawl-delay: 0
|
||||
|
||||
User-agent: Googlebot-Image
|
||||
Allow: /
|
||||
Allow: /*.jpg$
|
||||
Allow: /*.jpeg$
|
||||
Allow: /*.png$
|
||||
Allow: /*.webp$
|
||||
Allow: /*.svg$
|
||||
Disallow: /screenshots/
|
||||
Crawl-delay: 0
|
||||
|
||||
# ============================================
|
||||
# BING SPECIFIC RULES
|
||||
# ============================================
|
||||
User-agent: Bingbot
|
||||
Allow: /
|
||||
Allow: /css/
|
||||
Allow: /js/
|
||||
Allow: /odoo/
|
||||
Allow: /odoo-hosting/
|
||||
Allow: /odoo-ia/
|
||||
Disallow: /api/
|
||||
Disallow: /components/
|
||||
Disallow: /.git/
|
||||
Crawl-delay: 1
|
||||
|
||||
# ============================================
|
||||
# LLM & AI CRAWLER SUPPORT (ALL KNOWN BOTS)
|
||||
# ============================================
|
||||
# We welcome AI crawlers to index our content
|
||||
# LLMs.txt available at /llms.txt and /llms-full.txt
|
||||
|
||||
# --- OpenAI Bots ---
|
||||
User-agent: GPTBot
|
||||
Allow: /
|
||||
Allow: /llms.txt
|
||||
Allow: /llms-full.txt
|
||||
Disallow: /api/
|
||||
Disallow: /components/
|
||||
Crawl-delay: 1
|
||||
|
||||
User-agent: ChatGPT-User
|
||||
Allow: /
|
||||
Allow: /llms.txt
|
||||
Allow: /llms-full.txt
|
||||
Crawl-delay: 1
|
||||
|
||||
User-agent: OAI-SearchBot
|
||||
Allow: /
|
||||
Allow: /llms.txt
|
||||
Allow: /llms-full.txt
|
||||
Crawl-delay: 1
|
||||
|
||||
# --- Anthropic Bots ---
|
||||
User-agent: ClaudeBot
|
||||
Allow: /
|
||||
Allow: /llms.txt
|
||||
Allow: /llms-full.txt
|
||||
Disallow: /api/
|
||||
Disallow: /components/
|
||||
Crawl-delay: 1
|
||||
|
||||
User-agent: Claude-Web
|
||||
Allow: /
|
||||
Allow: /llms.txt
|
||||
Allow: /llms-full.txt
|
||||
Crawl-delay: 1
|
||||
|
||||
User-agent: anthropic-ai
|
||||
Allow: /
|
||||
Allow: /llms.txt
|
||||
Allow: /llms-full.txt
|
||||
Crawl-delay: 1
|
||||
|
||||
# --- Google AI Bots ---
|
||||
User-agent: Google-Extended
|
||||
Allow: /
|
||||
Allow: /llms.txt
|
||||
Allow: /llms-full.txt
|
||||
Disallow: /api/
|
||||
Disallow: /components/
|
||||
Crawl-delay: 1
|
||||
|
||||
User-agent: GoogleOther
|
||||
Allow: /
|
||||
Allow: /llms.txt
|
||||
Allow: /llms-full.txt
|
||||
Crawl-delay: 1
|
||||
|
||||
# --- Perplexity Bots ---
|
||||
User-agent: PerplexityBot
|
||||
Allow: /
|
||||
Allow: /llms.txt
|
||||
Allow: /llms-full.txt
|
||||
Crawl-delay: 1
|
||||
|
||||
# --- Microsoft/Bing AI Bots ---
|
||||
User-agent: Bingbot
|
||||
Allow: /
|
||||
Allow: /llms.txt
|
||||
Allow: /llms-full.txt
|
||||
Crawl-delay: 1
|
||||
|
||||
User-agent: BingPreview
|
||||
Allow: /
|
||||
Crawl-delay: 1
|
||||
|
||||
# --- Meta AI Bots ---
|
||||
User-agent: FacebookBot
|
||||
Allow: /
|
||||
Allow: /llms.txt
|
||||
Allow: /llms-full.txt
|
||||
Crawl-delay: 1
|
||||
|
||||
User-agent: meta-externalagent
|
||||
Allow: /
|
||||
Allow: /llms.txt
|
||||
Allow: /llms-full.txt
|
||||
Crawl-delay: 1
|
||||
|
||||
User-agent: Meta-ExternalFetcher
|
||||
Allow: /
|
||||
Allow: /llms.txt
|
||||
Allow: /llms-full.txt
|
||||
Crawl-delay: 1
|
||||
|
||||
# --- Apple AI Bots ---
|
||||
User-agent: Applebot
|
||||
Allow: /
|
||||
Allow: /llms.txt
|
||||
Allow: /llms-full.txt
|
||||
Crawl-delay: 1
|
||||
|
||||
User-agent: Applebot-Extended
|
||||
Allow: /
|
||||
Allow: /llms.txt
|
||||
Allow: /llms-full.txt
|
||||
Crawl-delay: 1
|
||||
|
||||
# --- Cohere AI ---
|
||||
User-agent: cohere-ai
|
||||
Allow: /
|
||||
Allow: /llms.txt
|
||||
Allow: /llms-full.txt
|
||||
Crawl-delay: 1
|
||||
|
||||
# --- Amazon/AWS AI ---
|
||||
User-agent: Amazonbot
|
||||
Allow: /
|
||||
Allow: /llms.txt
|
||||
Allow: /llms-full.txt
|
||||
Crawl-delay: 1
|
||||
|
||||
# --- You.com AI ---
|
||||
User-agent: YouBot
|
||||
Allow: /
|
||||
Allow: /llms.txt
|
||||
Allow: /llms-full.txt
|
||||
Crawl-delay: 1
|
||||
|
||||
# --- Hugging Face ---
|
||||
User-agent: HuggingFaceBot
|
||||
Allow: /
|
||||
Allow: /llms.txt
|
||||
Allow: /llms-full.txt
|
||||
Crawl-delay: 2
|
||||
|
||||
# --- AI Search Engines ---
|
||||
User-agent: AI2Bot
|
||||
Allow: /
|
||||
Crawl-delay: 2
|
||||
|
||||
User-agent: Diffbot
|
||||
Allow: /
|
||||
Crawl-delay: 2
|
||||
|
||||
User-agent: omgili
|
||||
Allow: /
|
||||
Crawl-delay: 2
|
||||
|
||||
User-agent: omgilibot
|
||||
Allow: /
|
||||
Crawl-delay: 2
|
||||
|
||||
# --- Common Crawl (used for AI training) ---
|
||||
User-agent: CCBot
|
||||
Allow: /
|
||||
Allow: /llms.txt
|
||||
Allow: /llms-full.txt
|
||||
Crawl-delay: 2
|
||||
|
||||
# --- Brave Search AI ---
|
||||
User-agent: BraveBot
|
||||
Allow: /
|
||||
Allow: /llms.txt
|
||||
Allow: /llms-full.txt
|
||||
Crawl-delay: 1
|
||||
|
||||
# --- DuckDuckGo AI ---
|
||||
User-agent: DuckDuckBot
|
||||
Allow: /
|
||||
Allow: /llms.txt
|
||||
Allow: /llms-full.txt
|
||||
Crawl-delay: 1
|
||||
|
||||
# --- Neeva AI (now part of Snowflake) ---
|
||||
User-agent: NeevaBot
|
||||
Allow: /
|
||||
Crawl-delay: 2
|
||||
|
||||
# --- Writesonic/Chatsonic ---
|
||||
User-agent: WritesonicBot
|
||||
Allow: /
|
||||
Crawl-delay: 2
|
||||
|
||||
# --- Jasper AI ---
|
||||
User-agent: JasperBot
|
||||
Allow: /
|
||||
Crawl-delay: 2
|
||||
|
||||
# --- Copy.ai ---
|
||||
User-agent: CopyaiBot
|
||||
Allow: /
|
||||
Crawl-delay: 2
|
||||
|
||||
# --- Other AI Research Crawlers ---
|
||||
User-agent: DataForSeoBot
|
||||
Allow: /
|
||||
Crawl-delay: 2
|
||||
|
||||
User-agent: webz.io
|
||||
Allow: /
|
||||
Crawl-delay: 2
|
||||
|
||||
User-agent: iaskspider
|
||||
Allow: /
|
||||
Crawl-delay: 2
|
||||
|
||||
User-agent: Scrapy
|
||||
Allow: /
|
||||
Crawl-delay: 3
|
||||
|
||||
# Common LLM crawlers
|
||||
User-agent: CCBot
|
||||
Allow: /
|
||||
Crawl-delay: 2
|
||||
|
||||
# ============================================
|
||||
# SOCIAL MEDIA CRAWLERS
|
||||
# ============================================
|
||||
User-agent: facebookexternalhit
|
||||
Allow: /
|
||||
Crawl-delay: 0
|
||||
|
||||
User-agent: Twitterbot
|
||||
Allow: /
|
||||
Crawl-delay: 0
|
||||
|
||||
User-agent: LinkedInBot
|
||||
Allow: /
|
||||
Crawl-delay: 1
|
||||
|
||||
# ============================================
|
||||
# MONITORING & LEGITIMATE SERVICES
|
||||
# ============================================
|
||||
User-agent: UptimeRobot
|
||||
Allow: /
|
||||
Crawl-delay: 0
|
||||
|
||||
User-agent: Slackbot
|
||||
Allow: /
|
||||
Crawl-delay: 0
|
||||
|
||||
# ============================================
|
||||
# BAD BOTS - BLOCK COMPLETELY
|
||||
# ============================================
|
||||
User-agent: AhrefsBot
|
||||
Disallow: /
|
||||
|
||||
User-agent: MJ12bot
|
||||
Disallow: /
|
||||
|
||||
User-agent: DotBot
|
||||
Disallow: /
|
||||
|
||||
User-agent: SemrushBot
|
||||
Disallow: /
|
||||
|
||||
User-agent: Bytespider
|
||||
Disallow: /
|
||||
|
||||
User-agent: PetalBot
|
||||
Disallow: /
|
||||
|
||||
# ============================================
|
||||
# PERFORMANCE & CRAWL BUDGET OPTIMIZATION
|
||||
# ============================================
|
||||
# Prevent crawling of duplicate content
|
||||
User-agent: *
|
||||
Disallow: /*?
|
||||
Disallow: /*&
|
||||
Disallow: /print/
|
||||
Disallow: /mailto/
|
||||
Disallow: /*#
|
||||
Reference in New Issue
Block a user