This commit is contained in:
oliver
2026-08-28 10:11:11 -03:00
parent f533ff8e36
commit 585951c8ce
1380 changed files with 1540268 additions and 0 deletions
+15
View File
@@ -0,0 +1,15 @@
"""CLI commands for Odoo Directory."""
from .setup import setup_command
from .plan import plan_command
from .execute import execute_command
from .export import export_command
from .deploy import deploy_command
__all__ = [
"setup_command",
"plan_command",
"execute_command",
"export_command",
"deploy_command",
]
+148
View File
@@ -0,0 +1,148 @@
"""
Deploy command for pushing changes to Git.
"""
import logging
import subprocess
from pathlib import Path
from datetime import datetime
from typing import Optional
import typer
from rich.console import Console
from ..config import get_settings
logger = logging.getLogger(__name__)
console = Console()
def deploy_command(
message: Optional[str] = None,
push: bool = True,
) -> None:
"""
Deploy exported files via Git.
Adds all changes in the export directory, commits them,
and optionally pushes to the remote repository.
Args:
message: Custom commit message (auto-generated if not provided)
push: Whether to push to remote after commit
"""
console.print("\n[bold blue]Deploying changes[/bold blue]\n")
settings = get_settings()
project_root = Path(settings.export_dir).parent
# Verify we're in a git repo
if not (project_root / ".git").exists():
console.print("[red]Not a git repository![/red]")
console.print(f"Expected .git directory in {project_root}")
raise typer.Exit(1)
try:
# Check for changes
result = subprocess.run(
["git", "status", "--porcelain"],
cwd=project_root,
capture_output=True,
text=True,
check=True,
)
if not result.stdout.strip():
console.print("[yellow]No changes to deploy.[/yellow]")
return
# Show changes
console.print("[bold]Changes to deploy:[/bold]")
console.print(result.stdout)
# Confirm deployment
if not typer.confirm("Deploy these changes?"):
console.print("[yellow]Deployment cancelled.[/yellow]")
raise typer.Exit(0)
# Generate commit message if not provided
if not message:
timestamp = datetime.now().strftime("%Y-%m-%d %H:%M")
message = f"Directory update: {timestamp}"
# Stage changes
console.print("Staging changes...")
subprocess.run(
["git", "add", "export/", "static/images/companies/"],
cwd=project_root,
check=True,
)
# Commit
console.print(f"Committing: {message}")
subprocess.run(
["git", "commit", "-m", message],
cwd=project_root,
check=True,
)
# Push if requested
if push:
console.print("Pushing to remote...")
subprocess.run(
["git", "push"],
cwd=project_root,
check=True,
)
console.print("[green]Changes pushed to remote.[/green]")
else:
console.print("[yellow]Changes committed locally but not pushed.[/yellow]")
console.print("\n[bold green]Deployment complete![/bold green]")
# Show deployment info
console.print("\nVercel will automatically deploy the changes.")
console.print("Check deployment status at: https://vercel.com/dashboard")
except subprocess.CalledProcessError as e:
console.print(f"[red]Git command failed: {e}[/red]")
if e.stderr:
console.print(f"Error: {e.stderr}")
raise typer.Exit(1)
except Exception as e:
console.print(f"[red]Deployment failed: {e}[/red]")
logger.exception("Deployment error")
raise typer.Exit(1)
def status_command() -> None:
"""Show git status of the project."""
settings = get_settings()
project_root = Path(settings.export_dir).parent
try:
# Git status
result = subprocess.run(
["git", "status"],
cwd=project_root,
capture_output=True,
text=True,
check=True,
)
console.print(result.stdout)
# Last commit
console.print("\n[bold]Last commit:[/bold]")
result = subprocess.run(
["git", "log", "-1", "--oneline"],
cwd=project_root,
capture_output=True,
text=True,
check=True,
)
console.print(result.stdout)
except subprocess.CalledProcessError as e:
console.print(f"[red]Git command failed: {e}[/red]")
raise typer.Exit(1)
+341
View File
@@ -0,0 +1,341 @@
"""
Execute command for running pipeline steps.
"""
import logging
from datetime import datetime
from typing import Optional, List
from uuid import UUID
import re
import typer
from rich.console import Console
from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn
from ..services.supabase import SupabaseService
from ..services.apify import ApifyService
from ..services.firecrawl import FirecrawlService
from ..services.haiku import HaikuService
from ..services.images import ImageService
from ..services.tags import TagAggregationService
from ..models.schemas import Company, StepStatus, TagCategory
logger = logging.getLogger(__name__)
console = Console()
def execute_command(
plan_id: str,
step: Optional[str] = None,
city: Optional[str] = None,
dry_run: bool = False,
) -> None:
"""
Execute an execution plan or specific step.
Runs the pipeline to scrape, enrich, and generate content for
Odoo partners.
Args:
plan_id: UUID of the execution plan
step: Specific step to run (scrape, enrich, content, export)
city: Specific city slug to process
dry_run: If True, don't make actual API calls
"""
console.print(f"\n[bold blue]Executing Plan: {plan_id}[/bold blue]\n")
db = SupabaseService()
try:
plan_uuid = UUID(plan_id)
except ValueError:
console.print(f"[red]Invalid plan ID: {plan_id}[/red]")
raise typer.Exit(1)
plan = db.get_execution_plan(plan_uuid)
if not plan:
console.print(f"[red]Plan not found: {plan_id}[/red]")
raise typer.Exit(1)
# Get country info
country = db.get_country_by_code(plan.country_code)
if not country:
console.print(f"[red]Country not found: {plan.country_code}[/red]")
raise typer.Exit(1)
# Filter cities if specified
city_slugs = plan.cities
if city:
if city not in city_slugs:
console.print(f"[red]City {city} not in plan[/red]")
raise typer.Exit(1)
city_slugs = [city]
# Determine steps to run
if step:
steps_to_run = [step]
else:
steps_to_run = ["scrape", "enrich", "content"]
console.print(f"Country: {country.name}")
console.print(f"Cities: {len(city_slugs)}")
console.print(f"Steps: {', '.join(steps_to_run)}")
console.print(f"Dry run: {dry_run}")
console.print()
# Update plan status
if not dry_run:
db.update_execution_plan(
plan_uuid, {"status": "running", "started_at": datetime.utcnow().isoformat()}
)
# Run each step
for step_name in steps_to_run:
console.print(f"\n[bold cyan]Running step: {step_name}[/bold cyan]\n")
for city_slug in city_slugs:
city_obj = db.get_city_by_slug(city_slug, country.id)
if not city_obj:
console.print(f"[yellow]City not found: {city_slug}[/yellow]")
continue
try:
if step_name == "scrape":
_run_scrape_step(db, city_obj, country, dry_run)
elif step_name == "enrich":
_run_enrich_step(db, city_obj, dry_run)
elif step_name == "content":
_run_content_step(db, city_obj, country, dry_run)
except Exception as e:
console.print(f"[red]Error in {step_name} for {city_slug}: {e}[/red]")
logger.exception(f"Step {step_name} failed for {city_slug}")
# Update plan status
if not dry_run:
db.update_execution_plan(
plan_uuid, {"status": "completed", "completed_at": datetime.utcnow().isoformat()}
)
console.print("\n[bold green]Execution complete![/bold green]")
def _run_scrape_step(db: SupabaseService, city, country, dry_run: bool) -> None:
"""Run the scrape step for a city."""
console.print(f"Scraping {city.name}...")
if dry_run:
console.print("[yellow]Dry run - skipping actual scrape[/yellow]")
return
apify = ApifyService()
# Scrape Google Maps
results = apify.scrape_city(
city_name=city.name,
language=country.language,
country=country.code,
)
# Deduplicate
results = apify.deduplicate_results(results)
console.print(f"Found {len(results)} unique companies")
# Save to database
saved = 0
for result in results:
# Check if already exists
if result.placeId:
existing = db.get_company_by_place_id(result.placeId)
if existing:
continue
# Create slug from name
slug = _create_slug(result.title)
# Extract location
lat = lng = None
if result.location:
lat = result.location.get("lat")
lng = result.location.get("lng")
company = Company(
city_id=city.id,
name=result.title,
slug=slug,
google_place_id=result.placeId,
address=result.address,
phone=result.phone,
website=result.website,
rating=result.rating,
review_count=result.reviewsCount,
latitude=lat,
longitude=lng,
status="scraped",
last_scraped_at=datetime.utcnow(),
)
db.create_company(company)
saved += 1
console.print(f"[green]Saved {saved} new companies[/green]")
def _run_enrich_step(db: SupabaseService, city, dry_run: bool) -> None:
"""Run the enrich step for companies in a city."""
console.print(f"Enriching companies in {city.name}...")
# Get companies that need enrichment
companies = db.list_companies(city_id=city.id, status="scraped", limit=100)
console.print(f"Found {len(companies)} companies to enrich")
if not companies or dry_run:
if dry_run:
console.print("[yellow]Dry run - skipping enrichment[/yellow]")
return
firecrawl = FirecrawlService()
with Progress(
SpinnerColumn(),
TextColumn("[progress.description]{task.description}"),
BarColumn(),
TextColumn("[progress.percentage]{task.percentage:>3.0f}%"),
console=console,
) as progress:
task = progress.add_task("Enriching...", total=len(companies))
for company in companies:
if not company.website:
progress.advance(task)
continue
try:
info = firecrawl.extract_company_info(company.website)
updates = {
"website_markdown": info.get("markdown"),
"logo_url": info.get("logo_url"),
"email": info.get("email") or company.email,
"status": "enriched",
"last_enriched_at": datetime.utcnow().isoformat(),
}
db.update_company(company.id, updates)
except Exception as e:
logger.warning(f"Failed to enrich {company.name}: {e}")
progress.advance(task)
console.print("[green]Enrichment complete[/green]")
def _run_content_step(db: SupabaseService, city, country, dry_run: bool) -> None:
"""Run the content generation step."""
console.print(f"Generating content for {city.name}...")
# Get companies that need content
companies = db.list_companies(city_id=city.id, status="enriched", limit=100)
console.print(f"Found {len(companies)} companies for content generation")
if not companies or dry_run:
if dry_run:
console.print("[yellow]Dry run - skipping content generation[/yellow]")
return
haiku = HaikuService()
tags_service = TagAggregationService(db)
with Progress(
SpinnerColumn(),
TextColumn("[progress.description]{task.description}"),
BarColumn(),
TextColumn("[progress.percentage]{task.percentage:>3.0f}%"),
console=console,
) as progress:
task = progress.add_task("Generating...", total=len(companies))
for company in companies:
try:
# Extract tags
tags_response = haiku.extract_tags(
company_name=company.name,
website_markdown=company.website_markdown,
address=company.address,
)
# Save tags
_save_company_tags(db, company.id, tags_response)
# Generate content
content = haiku.generate_company_content(
company_name=company.name,
city_name=city.name,
country_name=country.name,
website_markdown=company.website_markdown,
services=tags_response.services,
industries=tags_response.industries,
modules=tags_response.modules,
partner_level=tags_response.partner_level,
language=country.language,
)
updates = {
"description": content.description,
"content_html": content.content_html,
"meta_title": content.meta_title,
"meta_description": content.meta_description,
"status": "content_generated",
"last_content_at": datetime.utcnow().isoformat(),
}
db.update_company(company.id, updates)
except Exception as e:
logger.warning(f"Failed to generate content for {company.name}: {e}")
progress.advance(task)
# Aggregate tags for city
console.print("Aggregating city tags...")
tags_service.aggregate_city_tags(city.id)
console.print("[green]Content generation complete[/green]")
def _save_company_tags(db: SupabaseService, company_id: UUID, tags_response) -> None:
"""Save extracted tags to company."""
# Clear existing tags
db.remove_company_tags(company_id)
# Add service tags
for name in tags_response.services:
tag = db.get_or_create_tag(name, TagCategory.SERVICE)
db.add_company_tag(company_id, tag.id, tags_response.confidence)
# Add industry tags
for name in tags_response.industries:
tag = db.get_or_create_tag(name, TagCategory.INDUSTRY)
db.add_company_tag(company_id, tag.id, tags_response.confidence)
# Add module tags
for name in tags_response.modules:
tag = db.get_or_create_tag(name, TagCategory.MODULE)
db.add_company_tag(company_id, tag.id, tags_response.confidence)
# Add partner level if present
if tags_response.partner_level:
tag = db.get_or_create_tag(tags_response.partner_level, TagCategory.PARTNER_LEVEL)
db.add_company_tag(company_id, tag.id, tags_response.confidence)
def _create_slug(name: str) -> str:
"""Create URL-friendly slug from company name."""
# Convert to lowercase
slug = name.lower()
# Replace spaces and special chars with hyphens
slug = re.sub(r"[^a-z0-9]+", "-", slug)
# Remove leading/trailing hyphens
slug = slug.strip("-")
# Limit length
return slug[:100]
+511
View File
@@ -0,0 +1,511 @@
"""
Export command for generating static files.
"""
import json
import logging
from pathlib import Path
from typing import Optional, List, Dict, Any
from uuid import UUID
import typer
from rich.console import Console
from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn
from jinja2 import Environment, FileSystemLoader
from ..services.supabase import SupabaseService
from ..services.tags import TagAggregationService
from ..services.images import ImageService
from ..config import get_settings
logger = logging.getLogger(__name__)
console = Console()
def export_command(
country: str,
city: Optional[str] = None,
output_dir: Optional[str] = None,
) -> None:
"""
Export static XML/HTML files for a country or city.
Generates company pages, city hubs, country hubs, and sitemaps
from the database content.
Args:
country: Country code (e.g., DE)
city: Optional specific city slug
output_dir: Optional output directory override
"""
console.print(f"\n[bold blue]Exporting files for {country}[/bold blue]\n")
settings = get_settings()
db = SupabaseService()
tags_service = TagAggregationService(db)
image_service = ImageService()
# Setup output directory
if output_dir:
export_path = Path(output_dir)
else:
export_path = settings.export_dir
export_path.mkdir(parents=True, exist_ok=True)
# Get country info
country_obj = db.get_country_by_code(country.upper())
if not country_obj:
console.print(f"[red]Country not found: {country}[/red]")
raise typer.Exit(1)
# Setup Jinja environment
jinja_env = _setup_jinja_environment(settings.templates_dir)
# Get cities to export
all_cities = db.list_cities(country_id=country_obj.id)
if city:
cities_to_export = [c for c in all_cities if c.slug == city]
if not cities_to_export:
console.print(f"[red]City not found: {city}[/red]")
raise typer.Exit(1)
else:
cities_to_export = all_cities
console.print(f"Exporting {len(cities_to_export)} cities...")
# Export each city
total_companies = 0
with Progress(
SpinnerColumn(),
TextColumn("[progress.description]{task.description}"),
BarColumn(),
TextColumn("[progress.percentage]{task.percentage:>3.0f}%"),
console=console,
) as progress:
task = progress.add_task("Exporting...", total=len(cities_to_export))
for city_obj in cities_to_export:
city_companies = _export_city(
db=db,
tags_service=tags_service,
image_service=image_service,
jinja_env=jinja_env,
country=country_obj,
city=city_obj,
export_path=export_path,
)
total_companies += city_companies
progress.advance(task)
# Export country hub
_export_country_hub(
db=db,
tags_service=tags_service,
jinja_env=jinja_env,
country=country_obj,
cities=cities_to_export,
export_path=export_path,
)
# Export sitemaps
_export_sitemaps(
db=db,
country=country_obj,
cities=cities_to_export,
export_path=export_path,
)
# Export tag JSON files for frontend filtering
_export_tag_json(
db=db,
tags_service=tags_service,
country=country_obj,
cities=cities_to_export,
export_path=export_path,
)
console.print(f"\n[bold green]Export complete![/bold green]")
console.print(f" - Companies: {total_companies}")
console.print(f" - Cities: {len(cities_to_export)}")
console.print(f" - Output: {export_path}")
def _setup_jinja_environment(templates_dir: Path) -> Environment:
"""Setup Jinja2 templating environment."""
# Create templates directory if it doesn't exist
templates_dir.mkdir(parents=True, exist_ok=True)
# Create default templates if they don't exist
_ensure_templates_exist(templates_dir)
return Environment(
loader=FileSystemLoader(str(templates_dir)),
autoescape=True,
)
def _ensure_templates_exist(templates_dir: Path) -> None:
"""Create default templates if they don't exist."""
# Company template
company_template = templates_dir / "company.xml"
if not company_template.exists():
company_template.write_text(_DEFAULT_COMPANY_TEMPLATE)
# City hub template
city_template = templates_dir / "city_hub.xml"
if not city_template.exists():
city_template.write_text(_DEFAULT_CITY_TEMPLATE)
# Country hub template
country_template = templates_dir / "country_hub.xml"
if not country_template.exists():
country_template.write_text(_DEFAULT_COUNTRY_TEMPLATE)
def _export_city(
db: SupabaseService,
tags_service: TagAggregationService,
image_service: ImageService,
jinja_env: Environment,
country,
city,
export_path: Path,
) -> int:
"""Export all files for a city."""
# Create city directory
city_path = export_path / country.slug / city.slug
city_path.mkdir(parents=True, exist_ok=True)
# Get companies
companies = db.list_companies(
city_id=city.id,
status="content_generated",
limit=500,
)
# Export each company
company_template = jinja_env.get_template("company.xml")
for company in companies:
company_tags = db.get_company_tags(company.id)
context = {
"company": company,
"city": city,
"country": country,
"tags": company_tags,
"services": [t for t in company_tags if hasattr(t, "category") and t.category == "service"],
"industries": [t for t in company_tags if hasattr(t, "category") and t.category == "industry"],
"modules": [t for t in company_tags if hasattr(t, "category") and t.category == "module"],
}
content = company_template.render(**context)
company_file = city_path / f"{company.slug}.html"
company_file.write_text(content, encoding="utf-8")
# Export city hub
city_tags = tags_service.get_city_tags_by_category(city.id)
hub_template = jinja_env.get_template("city_hub.xml")
hub_context = {
"city": city,
"country": country,
"companies": companies,
"company_count": len(companies),
"tags": city_tags,
}
hub_content = hub_template.render(**hub_context)
hub_file = city_path / "index.html"
hub_file.write_text(hub_content, encoding="utf-8")
return len(companies)
def _export_country_hub(
db: SupabaseService,
tags_service: TagAggregationService,
jinja_env: Environment,
country,
cities: List,
export_path: Path,
) -> None:
"""Export country hub page."""
country_path = export_path / country.slug
country_path.mkdir(parents=True, exist_ok=True)
# Get aggregate data
total_companies = 0
city_data = []
for city in cities:
count = db.count_companies(city_id=city.id, status="content_generated")
total_companies += count
city_data.append({
"city": city,
"company_count": count,
})
# Sort cities by company count
city_data.sort(key=lambda x: x["company_count"], reverse=True)
country_tags = tags_service.get_country_tags_by_category(country.id)
# Render template
template = jinja_env.get_template("country_hub.xml")
content = template.render(
country=country,
cities=city_data,
city_count=len(cities),
company_count=total_companies,
tags=country_tags,
)
hub_file = country_path / "index.html"
hub_file.write_text(content, encoding="utf-8")
def _export_sitemaps(
db: SupabaseService,
country,
cities: List,
export_path: Path,
) -> None:
"""Export XML sitemaps."""
sitemap_path = export_path / "sitemaps"
sitemap_path.mkdir(parents=True, exist_ok=True)
base_url = "https://odoo-expertos.com"
# Company sitemap
urls = []
for city in cities:
companies = db.list_companies(
city_id=city.id,
status="content_generated",
limit=500,
)
for company in companies:
urls.append(
f"{base_url}/{country.slug}/{city.slug}/{company.slug}.html"
)
sitemap_xml = _generate_sitemap_xml(urls)
sitemap_file = sitemap_path / f"sitemap-companies-{country.code.lower()}.xml"
sitemap_file.write_text(sitemap_xml, encoding="utf-8")
# City sitemap
city_urls = [
f"{base_url}/{country.slug}/{city.slug}/"
for city in cities
]
city_sitemap = _generate_sitemap_xml(city_urls)
city_sitemap_file = sitemap_path / f"sitemap-cities-{country.code.lower()}.xml"
city_sitemap_file.write_text(city_sitemap, encoding="utf-8")
def _generate_sitemap_xml(urls: List[str]) -> str:
"""Generate sitemap XML content."""
lines = ['<?xml version="1.0" encoding="UTF-8"?>']
lines.append('<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">')
for url in urls:
lines.append(" <url>")
lines.append(f" <loc>{url}</loc>")
lines.append(" <changefreq>monthly</changefreq>")
lines.append(" <priority>0.7</priority>")
lines.append(" </url>")
lines.append("</urlset>")
return "\n".join(lines)
def _export_tag_json(
db: SupabaseService,
tags_service: TagAggregationService,
country,
cities: List,
export_path: Path,
) -> None:
"""Export tag aggregations as JSON for frontend filtering."""
json_path = export_path / "tags"
json_path.mkdir(parents=True, exist_ok=True)
# City tags
cities_path = json_path / "cities"
cities_path.mkdir(parents=True, exist_ok=True)
for city in cities:
city_json = tags_service.export_city_tags_json(city.id)
json_file = cities_path / f"{city.slug}.json"
json_file.write_text(json.dumps(city_json, indent=2), encoding="utf-8")
# Country tags
country_json = tags_service.export_country_tags_json(country.id)
country_file = json_path / f"{country.slug}.json"
country_file.write_text(json.dumps(country_json, indent=2), encoding="utf-8")
# Default templates
_DEFAULT_COMPANY_TEMPLATE = """<!DOCTYPE html>
<html lang="{{ country.language }}">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>{{ company.meta_title or company.name }}</title>
<meta name="description" content="{{ company.meta_description or company.description }}">
<link rel="canonical" href="https://odoo-expertos.com/{{ country.slug }}/{{ city.slug }}/{{ company.slug }}.html">
</head>
<body>
<header>
<nav>
<a href="/{{ country.slug }}/">{{ country.name }}</a> &gt;
<a href="/{{ country.slug }}/{{ city.slug }}/">{{ city.name }}</a> &gt;
{{ company.name }}
</nav>
</header>
<main>
<article>
<h1>{{ company.name }}</h1>
{% if company.description %}
<p class="lead">{{ company.description }}</p>
{% endif %}
<div class="company-info">
{% if company.address %}
<p><strong>Address:</strong> {{ company.address }}</p>
{% endif %}
{% if company.phone %}
<p><strong>Phone:</strong> {{ company.phone }}</p>
{% endif %}
{% if company.website %}
<p><strong>Website:</strong> <a href="{{ company.website }}" rel="noopener">{{ company.website }}</a></p>
{% endif %}
{% if company.email %}
<p><strong>Email:</strong> {{ company.email }}</p>
{% endif %}
{% if company.rating %}
<p><strong>Rating:</strong> {{ company.rating }}/5 ({{ company.review_count }} reviews)</p>
{% endif %}
</div>
<div class="content">
{{ company.content_html | safe }}
</div>
</article>
</main>
<footer>
<p>&copy; Odoo Expertos - {{ city.name }}, {{ country.name }}</p>
</footer>
</body>
</html>
"""
_DEFAULT_CITY_TEMPLATE = """<!DOCTYPE html>
<html lang="{{ country.language }}">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Odoo Partners in {{ city.name }} | {{ country.name }}</title>
<meta name="description" content="Find {{ company_count }} certified Odoo partners in {{ city.name }}, {{ country.name }}. Compare services, expertise, and reviews.">
<link rel="canonical" href="https://odoo-expertos.com/{{ country.slug }}/{{ city.slug }}/">
</head>
<body>
<header>
<nav>
<a href="/{{ country.slug }}/">{{ country.name }}</a> &gt;
{{ city.name }}
</nav>
</header>
<main>
<h1>Odoo Partners in {{ city.name }}</h1>
<p>Discover {{ company_count }} certified Odoo partners in {{ city.name }}, {{ country.name }}.</p>
<section class="filters">
<h2>Filter by Services</h2>
<ul>
{% for tag in tags.services %}
<li>{{ tag.name }} ({{ tag.count }})</li>
{% endfor %}
</ul>
</section>
<section class="companies">
<h2>Odoo Partners</h2>
{% for company in companies %}
<article class="company-card">
<h3><a href="{{ company.slug }}.html">{{ company.name }}</a></h3>
{% if company.description %}
<p>{{ company.description }}</p>
{% endif %}
{% if company.rating %}
<p class="rating">{{ company.rating }}/5</p>
{% endif %}
</article>
{% endfor %}
</section>
</main>
<footer>
<p>&copy; Odoo Expertos - {{ city.name }}, {{ country.name }}</p>
</footer>
</body>
</html>
"""
_DEFAULT_COUNTRY_TEMPLATE = """<!DOCTYPE html>
<html lang="{{ country.language }}">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Odoo Partners in {{ country.name }}</title>
<meta name="description" content="Find {{ company_count }} certified Odoo partners in {{ city_count }} cities across {{ country.name }}.">
<link rel="canonical" href="https://odoo-expertos.com/{{ country.slug }}/">
</head>
<body>
<header>
<nav>
{{ country.name }}
</nav>
</header>
<main>
<h1>Odoo Partners in {{ country.name }}</h1>
<p>Discover {{ company_count }} certified Odoo partners in {{ city_count }} cities across {{ country.name }}.</p>
<section class="cities">
<h2>Cities</h2>
<ul>
{% for item in cities %}
<li>
<a href="{{ item.city.slug }}/">{{ item.city.name }}</a>
({{ item.company_count }} partners)
</li>
{% endfor %}
</ul>
</section>
<section class="filters">
<h2>Services</h2>
<ul>
{% for tag in tags.services %}
<li>{{ tag.name }} ({{ tag.company_count }} companies in {{ tag.city_count }} cities)</li>
{% endfor %}
</ul>
</section>
</main>
<footer>
<p>&copy; Odoo Expertos - {{ country.name }}</p>
</footer>
</body>
</html>
"""
+242
View File
@@ -0,0 +1,242 @@
"""
Plan command for creating execution plans.
"""
import logging
from typing import List, Optional
from uuid import UUID
import typer
from rich.console import Console
from rich.table import Table
from ..services.supabase import SupabaseService
from ..models.schemas import ExecutionPlan, PlanStep, StepStatus
from ..config import SEARCH_QUERIES
logger = logging.getLogger(__name__)
console = Console()
def plan_command(
country: str,
cities: Optional[List[str]] = None,
name: Optional[str] = None,
) -> Optional[UUID]:
"""
Create an execution plan for scraping Odoo partners.
This generates a plan that specifies which cities to scrape and
what steps to perform. The plan can then be executed with the
execute command.
Args:
country: Country code (e.g., DE for Germany)
cities: List of city slugs to include (default: all cities in country)
name: Optional name for the plan
Returns:
Plan UUID if successful
"""
console.print(f"\n[bold blue]Creating Execution Plan for {country}[/bold blue]\n")
db = SupabaseService()
# Verify country exists
country_obj = db.get_country_by_code(country.upper())
if not country_obj:
console.print(f"[red]Country not found: {country}[/red]")
console.print("Run 'setup' command first to seed country data.")
raise typer.Exit(1)
# Get cities
all_cities = db.list_cities(country_id=country_obj.id)
if cities:
# Filter to specified cities
city_slugs = [c.lower() for c in cities]
selected_cities = [c for c in all_cities if c.slug in city_slugs]
missing = set(city_slugs) - {c.slug for c in selected_cities}
if missing:
console.print(f"[yellow]Warning: Cities not found: {missing}[/yellow]")
else:
selected_cities = all_cities
if not selected_cities:
console.print("[red]No cities found for this country.[/red]")
raise typer.Exit(1)
# Generate plan name
if not name:
city_names = [c.slug for c in selected_cities[:3]]
name = f"{country.upper()}-{'-'.join(city_names)}"
if len(selected_cities) > 3:
name += f"-and-{len(selected_cities) - 3}-more"
# Estimate company counts based on queries
language = country_obj.language
queries_per_city = len(SEARCH_QUERIES.get(language, SEARCH_QUERIES["en"]))
estimated_companies = len(selected_cities) * queries_per_city * 15 # ~15 per query
# Show plan summary
table = Table(title="Execution Plan Summary")
table.add_column("Field", style="cyan")
table.add_column("Value", style="green")
table.add_row("Plan Name", name)
table.add_row("Country", f"{country_obj.name} ({country_obj.code})")
table.add_row("Language", country_obj.language)
table.add_row("Cities", str(len(selected_cities)))
table.add_row("Queries/City", str(queries_per_city))
table.add_row("Est. Companies", f"~{estimated_companies}")
console.print(table)
console.print()
# Show city list
console.print("[bold]Cities included:[/bold]")
for city in selected_cities:
console.print(f" - {city.name} ({city.slug})")
console.print()
# Confirm creation
if not typer.confirm("Create this plan?"):
console.print("[yellow]Plan creation cancelled.[/yellow]")
raise typer.Exit(0)
# Create the plan
plan = ExecutionPlan(
name=name,
country_code=country.upper(),
cities=[c.slug for c in selected_cities],
status=StepStatus.PENDING,
)
plan = db.create_execution_plan(plan)
console.print(f"\n[green]Plan created: {plan.id}[/green]")
# Create steps for each city
step_types = ["scrape", "enrich", "content", "export"]
for city in selected_cities:
for step_type in step_types:
step = PlanStep(
plan_id=plan.id,
step_name=step_type,
city_id=city.id,
status=StepStatus.PENDING,
)
db.create_plan_step(step)
console.print(
f"[green]Created {len(selected_cities) * len(step_types)} steps[/green]"
)
# Print next steps
console.print("\n[bold]Next steps:[/bold]")
console.print(f" 1. Run: python -m directory.cli execute --plan {plan.id}")
console.print(" 2. Or run step by step:")
console.print(f" python -m directory.cli execute --plan {plan.id} --step scrape")
console.print(
f" python -m directory.cli execute --plan {plan.id} --step enrich"
)
console.print(
f" python -m directory.cli execute --plan {plan.id} --step content"
)
console.print()
return plan.id
def list_plans() -> None:
"""List all execution plans."""
db = SupabaseService()
plans = db.list_execution_plans()
if not plans:
console.print("[yellow]No execution plans found.[/yellow]")
return
table = Table(title="Execution Plans")
table.add_column("ID", style="cyan", max_width=36)
table.add_column("Name", style="green")
table.add_column("Country", style="blue")
table.add_column("Cities", style="magenta")
table.add_column("Status", style="yellow")
table.add_column("Created", style="dim")
for plan in plans:
table.add_row(
str(plan.id),
plan.name,
plan.country_code,
str(len(plan.cities)),
plan.status.value if hasattr(plan.status, "value") else plan.status,
str(plan.created_at)[:16] if plan.created_at else "-",
)
console.print(table)
def show_plan(plan_id: str) -> None:
"""Show details of a specific plan."""
db = SupabaseService()
try:
plan_uuid = UUID(plan_id)
except ValueError:
console.print(f"[red]Invalid plan ID: {plan_id}[/red]")
raise typer.Exit(1)
plan = db.get_execution_plan(plan_uuid)
if not plan:
console.print(f"[red]Plan not found: {plan_id}[/red]")
raise typer.Exit(1)
# Plan info
console.print(f"\n[bold blue]Plan: {plan.name}[/bold blue]\n")
table = Table()
table.add_column("Field", style="cyan")
table.add_column("Value", style="green")
table.add_row("ID", str(plan.id))
table.add_row("Country", plan.country_code)
table.add_row("Cities", ", ".join(plan.cities))
table.add_row("Status", plan.status.value if hasattr(plan.status, "value") else plan.status)
table.add_row("Created", str(plan.created_at)[:19] if plan.created_at else "-")
table.add_row("Started", str(plan.started_at)[:19] if plan.started_at else "-")
table.add_row("Completed", str(plan.completed_at)[:19] if plan.completed_at else "-")
console.print(table)
# Steps
if plan.steps:
console.print("\n[bold]Steps:[/bold]")
steps_table = Table()
steps_table.add_column("Step", style="cyan")
steps_table.add_column("City", style="blue")
steps_table.add_column("Status", style="yellow")
steps_table.add_column("Progress", style="green")
for step in plan.steps:
city_slug = "-"
if step.city_id:
city = db.get_city_by_slug("", step.city_id)
# We don't have a get_city_by_id, so we'll just show the ID
city_slug = str(step.city_id)[:8]
progress = f"{step.items_processed}/{step.items_total}" if step.items_total else "-"
steps_table.add_row(
step.step_name,
city_slug,
step.status.value if hasattr(step.status, "value") else step.status,
progress,
)
console.print(steps_table)
+304
View File
@@ -0,0 +1,304 @@
"""
Setup command for initializing the database.
"""
import logging
from typing import Optional
import typer
from rich.console import Console
from rich.progress import Progress, SpinnerColumn, TextColumn
from ..services.supabase import SupabaseService
from ..models.schemas import Region, Country, City, Tag, TagCategory
from ..config import DEFAULT_TAGS, GERMANY_CITIES
logger = logging.getLogger(__name__)
console = Console()
# SQL schema for creating tables
SCHEMA_SQL = """
-- Enable UUID extension
CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
-- Regions table
CREATE TABLE IF NOT EXISTS regions (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
name TEXT NOT NULL,
slug TEXT NOT NULL UNIQUE,
created_at TIMESTAMPTZ DEFAULT NOW()
);
-- Countries table
CREATE TABLE IF NOT EXISTS countries (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
region_id UUID REFERENCES regions(id),
name TEXT NOT NULL,
name_local TEXT,
code TEXT NOT NULL UNIQUE,
slug TEXT NOT NULL UNIQUE,
language TEXT NOT NULL,
created_at TIMESTAMPTZ DEFAULT NOW()
);
-- Cities table
CREATE TABLE IF NOT EXISTS cities (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
country_id UUID REFERENCES countries(id),
name TEXT NOT NULL,
name_local TEXT,
slug TEXT NOT NULL,
state TEXT,
population INTEGER,
latitude FLOAT,
longitude FLOAT,
created_at TIMESTAMPTZ DEFAULT NOW(),
UNIQUE(country_id, slug)
);
-- Tags table
CREATE TABLE IF NOT EXISTS tags (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
category TEXT NOT NULL,
name TEXT NOT NULL,
slug TEXT NOT NULL UNIQUE,
description TEXT,
created_at TIMESTAMPTZ DEFAULT NOW()
);
-- Companies table
CREATE TABLE IF NOT EXISTS companies (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
city_id UUID REFERENCES cities(id),
name TEXT NOT NULL,
slug TEXT NOT NULL,
google_place_id TEXT UNIQUE,
address TEXT,
phone TEXT,
website TEXT,
email TEXT,
rating FLOAT,
review_count INTEGER,
latitude FLOAT,
longitude FLOAT,
website_markdown TEXT,
logo_url TEXT,
description TEXT,
content_html TEXT,
meta_title TEXT,
meta_description TEXT,
status TEXT DEFAULT 'scraped',
last_scraped_at TIMESTAMPTZ,
last_enriched_at TIMESTAMPTZ,
last_content_at TIMESTAMPTZ,
created_at TIMESTAMPTZ DEFAULT NOW(),
updated_at TIMESTAMPTZ DEFAULT NOW()
);
-- Company tags junction table
CREATE TABLE IF NOT EXISTS company_tags (
company_id UUID REFERENCES companies(id) ON DELETE CASCADE,
tag_id UUID REFERENCES tags(id) ON DELETE CASCADE,
confidence FLOAT DEFAULT 1.0,
created_at TIMESTAMPTZ DEFAULT NOW(),
PRIMARY KEY (company_id, tag_id)
);
-- City tag aggregation
CREATE TABLE IF NOT EXISTS city_tags (
city_id UUID REFERENCES cities(id) ON DELETE CASCADE,
tag_id UUID REFERENCES tags(id) ON DELETE CASCADE,
company_count INTEGER DEFAULT 0,
PRIMARY KEY (city_id, tag_id)
);
-- Country tag aggregation
CREATE TABLE IF NOT EXISTS country_tags (
country_id UUID REFERENCES countries(id) ON DELETE CASCADE,
tag_id UUID REFERENCES tags(id) ON DELETE CASCADE,
company_count INTEGER DEFAULT 0,
city_count INTEGER DEFAULT 0,
PRIMARY KEY (country_id, tag_id)
);
-- Execution plans
CREATE TABLE IF NOT EXISTS execution_plans (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
name TEXT NOT NULL,
country_code TEXT NOT NULL,
cities TEXT[] NOT NULL,
status TEXT DEFAULT 'pending',
created_at TIMESTAMPTZ DEFAULT NOW(),
started_at TIMESTAMPTZ,
completed_at TIMESTAMPTZ
);
-- Plan steps
CREATE TABLE IF NOT EXISTS plan_steps (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
plan_id UUID REFERENCES execution_plans(id) ON DELETE CASCADE,
step_name TEXT NOT NULL,
city_id UUID REFERENCES cities(id),
status TEXT DEFAULT 'pending',
started_at TIMESTAMPTZ,
completed_at TIMESTAMPTZ,
error_message TEXT,
items_processed INTEGER DEFAULT 0,
items_total INTEGER DEFAULT 0
);
-- Error logs
CREATE TABLE IF NOT EXISTS error_logs (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
entity_type TEXT NOT NULL,
entity_id UUID,
operation TEXT NOT NULL,
error_message TEXT NOT NULL,
details JSONB DEFAULT '{}',
created_at TIMESTAMPTZ DEFAULT NOW()
);
-- Indexes for performance
CREATE INDEX IF NOT EXISTS idx_companies_city ON companies(city_id);
CREATE INDEX IF NOT EXISTS idx_companies_status ON companies(status);
CREATE INDEX IF NOT EXISTS idx_company_tags_company ON company_tags(company_id);
CREATE INDEX IF NOT EXISTS idx_company_tags_tag ON company_tags(tag_id);
CREATE INDEX IF NOT EXISTS idx_city_tags_city ON city_tags(city_id);
CREATE INDEX IF NOT EXISTS idx_country_tags_country ON country_tags(country_id);
CREATE INDEX IF NOT EXISTS idx_plan_steps_plan ON plan_steps(plan_id);
"""
def setup_command(
seed_tags: bool = True,
seed_germany: bool = True,
) -> None:
"""
Initialize the database with schema and seed data.
This command should be run once when setting up a new installation.
It creates all required tables and optionally seeds reference data.
Args:
seed_tags: Whether to seed default tags
seed_germany: Whether to seed Germany cities
"""
console.print("\n[bold blue]Odoo Directory Setup[/bold blue]\n")
db = SupabaseService()
with Progress(
SpinnerColumn(),
TextColumn("[progress.description]{task.description}"),
console=console,
) as progress:
# Note: Schema should be created in Supabase SQL Editor
# The CLI just verifies connectivity and seeds data
task = progress.add_task("Verifying database connection...", total=None)
try:
# Test connection by listing regions
regions = db.list_regions()
progress.update(task, description="[green]Database connected")
except Exception as e:
console.print(f"\n[red]Database connection failed: {e}[/red]")
console.print(
"\n[yellow]Please create tables using the SQL schema in Supabase SQL Editor.[/yellow]"
)
console.print("\nSchema SQL can be found in directory/commands/setup.py")
raise typer.Exit(1)
# Seed tags
if seed_tags:
task = progress.add_task("Seeding tags...", total=None)
tag_count = _seed_tags(db)
progress.update(task, description=f"[green]Seeded {tag_count} tags")
# Seed Germany
if seed_germany:
task = progress.add_task("Seeding Germany data...", total=None)
city_count = _seed_germany(db)
progress.update(task, description=f"[green]Seeded {city_count} cities")
console.print("\n[bold green]Setup complete![/bold green]\n")
def _seed_tags(db: SupabaseService) -> int:
"""Seed default tags into database."""
count = 0
# Map category names to enum values
category_map = {
"services": TagCategory.SERVICE,
"industries": TagCategory.INDUSTRY,
"modules": TagCategory.MODULE,
"partner_levels": TagCategory.PARTNER_LEVEL,
}
for category_name, tag_names in DEFAULT_TAGS.items():
category = category_map[category_name]
for name in tag_names:
slug = name.lower().replace(" ", "-").replace("&", "and")
existing = db.get_tag_by_slug(slug)
if not existing:
db.create_tag(
Tag(
category=category,
name=name,
slug=slug,
)
)
count += 1
return count
def _seed_germany(db: SupabaseService) -> int:
"""Seed Germany region, country, and cities."""
# Create Europe region if not exists
region = db.get_region_by_slug("europe")
if not region:
region = db.create_region(Region(name="Europe", slug="europe"))
# Create Germany if not exists
country = db.get_country_by_code("DE")
if not country:
country = db.create_country(
Country(
region_id=region.id,
name="Germany",
name_local="Deutschland",
code="DE",
slug="deutschland",
language="de",
)
)
# Create cities
count = 0
for city_data in GERMANY_CITIES:
existing = db.get_city_by_slug(city_data["slug"], country.id)
if not existing:
db.create_city(
City(
country_id=country.id,
name=city_data["name"],
slug=city_data["slug"],
state=city_data.get("state"),
)
)
count += 1
return count
def print_schema() -> None:
"""Print the SQL schema for manual creation."""
console.print("\n[bold]Database Schema SQL:[/bold]\n")
console.print(SCHEMA_SQL)
console.print(
"\n[yellow]Copy this SQL and run it in your Supabase SQL Editor.[/yellow]"
)