"""Depth + duplicate title/meta analysis for odoo-expertos.com (read-only)."""
import json, re, sys, urllib.parse as up
import requests
from bs4 import BeautifulSoup

BASE = "https://www.odoo-expertos.com"
S = requests.Session()
S.headers["User-Agent"] = "Mozilla/5.0 (SEO-audit; +read-only)"

def get(u):
    try:
        r = S.get(u, timeout=30)
        return r.status_code, r.text
    except Exception as e:
        return None, str(e)

def meta(html):
    soup = BeautifulSoup(html, "lxml")
    t = (soup.title.string.strip() if soup.title and soup.title.string else "")
    md = ""
    m = soup.find("meta", attrs={"name": "description"})
    if m and m.get("content"): md = m["content"].strip()
    h1s = [h.get_text(strip=True) for h in soup.find_all("h1")]
    return t, md, h1s, soup

def internal_links(soup):
    out = set()
    for a in soup.find_all("a", href=True):
        href = a["href"].split("#")[0].strip()
        if not href or href.startswith(("mailto:", "tel:", "javascript:")): continue
        full = up.urljoin(BASE + "/", href)
        if full.startswith(BASE):
            out.add(full.rstrip("/") + "/")
    return out

# 1. CLICK DEPTH (Spanish tree): home -> categories -> articles
depth = {}
_, home = get(BASE + "/")
soup_home = BeautifulSoup(home, "lxml") if home else None
d1 = internal_links(soup_home) if soup_home else set()
cats = [u for u in d1 if re.search(r"/(odoo|odoo-hosting|odoo-ia)/$", u)]
depth["home_internal_links"] = len(d1)
depth["categories_linked_from_home"] = sorted(cats)

d2 = set()
cat_link_counts = {}
for c in ["/odoo/", "/odoo-hosting/", "/odoo-ia/"]:
    code, html = get(BASE + c)
    if html and code == 200:
        s = BeautifulSoup(html, "lxml")
        links = internal_links(s)
        arts = {u for u in links if re.search(r"/(odoo|odoo-hosting|odoo-ia)/[^/]+/$", u)}
        cat_link_counts[c] = len(arts)
        d2 |= arts
depth["articles_linked_from_category_indexes(depth2)"] = len(d2)
depth["per_category_article_links"] = cat_link_counts

# Compare to total ES articles in sitemap
try:
    es = open(r"C:\Users\eugen\.cursor\projects\Odoo Expertos\odoo-expertos\seo-audit-2026-05-27\raw\sitemap-es.xml", encoding="utf-8").read()
    es_locs = set(re.findall(r"<loc>(https[^<]+)</loc>", es))
    es_articles = {u.rstrip("/") + "/" for u in es_locs if re.search(r"/(odoo|odoo-hosting|odoo-ia)/[^/]+/$", u)}
    depth["total_ES_articles_in_sitemap"] = len(es_articles)
    reachable = (d2 & es_articles)
    depth["ES_articles_reachable_within_2_clicks"] = len(reachable)
    depth["ES_articles_NOT_reachable_within_2_clicks"] = len(es_articles - reachable)
except Exception as e:
    depth["sitemap_compare_error"] = str(e)

# 2. DUPLICATE / IDENTICAL title + meta across a sample
sample = [
    "/", "/odoo/", "/odoo-hosting/", "/odoo-ia/",
    "/odoo/odoo-hosting-cloud-infraestructura-2025/",
    "/odoo/odoo-crm-gestion-clientes-2025/",
    "/odoo/odoo-inventario-gestion-almacen-2025/",
    "/odoo-hosting/aws-hosting-odoo-2026/",
    "/odoo-hosting/odoo-docker-hosting-2026/",
    "/odoo-hosting/servidor-dedicado-odoo-2026/",
    "/odoo-ia/que-es-odoo-ia-guia-basica-2025/",
    "/odoo-ia/odoo-openai-api-integracion-2025/",
    "/en/", "/de/", "/fr/", "/pt/", "/ar/",
]
rows = []
for path in sample:
    code, html = get(BASE + path)
    if html and code == 200:
        t, md, h1s, _ = meta(html)
        rows.append({
            "url": path, "status": code,
            "title": t, "title_len": len(t),
            "meta": md, "meta_len": len(md),
            "h1": h1s[0] if h1s else "", "h1_count": len(h1s),
            "title_eq_h1": (t.split("|")[0].strip().lower() == (h1s[0].lower() if h1s else "")),
            "title_eq_meta": (t.strip().lower() == md.strip().lower()),
        })
    else:
        rows.append({"url": path, "status": code})

# duplicate detection within sample
titles = {}
metas = {}
for r in rows:
    if r.get("title"): titles.setdefault(r["title"], []).append(r["url"])
    if r.get("meta"): metas.setdefault(r["meta"], []).append(r["url"])
dup_titles = {k: v for k, v in titles.items() if len(v) > 1}
dup_metas = {k: v for k, v in metas.items() if len(v) > 1}

result = {
    "depth": depth,
    "pages": rows,
    "duplicate_titles_in_sample": dup_titles,
    "duplicate_metas_in_sample": dup_metas,
}
print(json.dumps(result, ensure_ascii=False, indent=2))