new site
This commit is contained in:
oliver
@@ -0,0 +1,8 @@
|
||||
"""
|
||||
Odoo Directory CLI
|
||||
|
||||
A Python CLI tool that generates static XML/HTML pages for Odoo partners,
|
||||
integrating with the existing odoo-expertos.com static website.
|
||||
"""
|
||||
|
||||
__version__ = "0.1.0"
|
||||
@@ -0,0 +1,8 @@
|
||||
"""
|
||||
Allow running the package as a module: python -m directory
|
||||
"""
|
||||
|
||||
from .cli import main
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,225 @@
|
||||
"""
|
||||
Odoo Directory CLI - Main entry point.
|
||||
|
||||
A Python CLI tool that generates static XML/HTML pages for Odoo partners,
|
||||
integrating with the existing odoo-expertos.com static website.
|
||||
|
||||
Usage:
|
||||
python -m directory.cli setup
|
||||
python -m directory.cli plan --country DE --cities berlin muenchen
|
||||
python -m directory.cli execute --plan <plan-id>
|
||||
python -m directory.cli export --country DE
|
||||
python -m directory.cli deploy
|
||||
"""
|
||||
|
||||
import logging
|
||||
from typing import List, Optional
|
||||
|
||||
import typer
|
||||
from rich.console import Console
|
||||
|
||||
from .config import get_settings
|
||||
from .utils.logger import setup_logging
|
||||
from .commands.setup import setup_command, print_schema
|
||||
from .commands.plan import plan_command, list_plans, show_plan
|
||||
from .commands.execute import execute_command
|
||||
from .commands.export import export_command
|
||||
from .commands.deploy import deploy_command, status_command
|
||||
|
||||
# Create CLI app
|
||||
app = typer.Typer(
|
||||
name="odoo-directory",
|
||||
help="Generate static pages for Odoo partner directory.",
|
||||
add_completion=False,
|
||||
)
|
||||
|
||||
console = Console()
|
||||
|
||||
|
||||
# ==================== Setup Commands ====================
|
||||
|
||||
@app.command()
|
||||
def setup(
|
||||
seed_tags: bool = typer.Option(True, help="Seed default tags"),
|
||||
seed_germany: bool = typer.Option(True, help="Seed Germany cities"),
|
||||
schema: bool = typer.Option(False, "--schema", help="Print SQL schema only"),
|
||||
):
|
||||
"""
|
||||
Initialize database with schema and seed data.
|
||||
|
||||
Run this once when setting up a new installation.
|
||||
"""
|
||||
setup_logging()
|
||||
|
||||
if schema:
|
||||
print_schema()
|
||||
return
|
||||
|
||||
setup_command(seed_tags=seed_tags, seed_germany=seed_germany)
|
||||
|
||||
|
||||
# ==================== Plan Commands ====================
|
||||
|
||||
@app.command()
|
||||
def plan(
|
||||
country: str = typer.Option(..., "--country", "-c", help="Country code (e.g., DE)"),
|
||||
cities: Optional[List[str]] = typer.Option(None, "--cities", help="City slugs to include"),
|
||||
name: Optional[str] = typer.Option(None, "--name", "-n", help="Plan name"),
|
||||
):
|
||||
"""
|
||||
Create an execution plan for a country.
|
||||
|
||||
Specify which cities to process with --cities or process all cities
|
||||
in the country by default.
|
||||
"""
|
||||
setup_logging()
|
||||
plan_command(country=country, cities=cities, name=name)
|
||||
|
||||
|
||||
@app.command("plans")
|
||||
def plans_list():
|
||||
"""List all execution plans."""
|
||||
setup_logging()
|
||||
list_plans()
|
||||
|
||||
|
||||
@app.command("plan-show")
|
||||
def plan_show(
|
||||
plan_id: str = typer.Argument(..., help="Plan UUID"),
|
||||
):
|
||||
"""Show details of a specific plan."""
|
||||
setup_logging()
|
||||
show_plan(plan_id)
|
||||
|
||||
|
||||
# ==================== Execute Commands ====================
|
||||
|
||||
@app.command()
|
||||
def execute(
|
||||
plan_id: str = typer.Option(..., "--plan", "-p", help="Plan UUID"),
|
||||
step: Optional[str] = typer.Option(None, "--step", "-s", help="Specific step: scrape, enrich, content"),
|
||||
city: Optional[str] = typer.Option(None, "--city", help="Specific city slug"),
|
||||
dry_run: bool = typer.Option(False, "--dry-run", help="Don't make actual API calls"),
|
||||
):
|
||||
"""
|
||||
Execute a plan or specific step.
|
||||
|
||||
Runs the pipeline: scrape -> enrich -> content generation.
|
||||
"""
|
||||
setup_logging()
|
||||
execute_command(plan_id=plan_id, step=step, city=city, dry_run=dry_run)
|
||||
|
||||
|
||||
# ==================== Export Commands ====================
|
||||
|
||||
@app.command()
|
||||
def export(
|
||||
country: str = typer.Option(..., "--country", "-c", help="Country code"),
|
||||
city: Optional[str] = typer.Option(None, "--city", help="Specific city slug"),
|
||||
output: Optional[str] = typer.Option(None, "--output", "-o", help="Output directory"),
|
||||
):
|
||||
"""
|
||||
Export static files for a country.
|
||||
|
||||
Generates HTML pages, sitemaps, and tag JSON files.
|
||||
"""
|
||||
setup_logging()
|
||||
export_command(country=country, city=city, output_dir=output)
|
||||
|
||||
|
||||
# ==================== Deploy Commands ====================
|
||||
|
||||
@app.command()
|
||||
def deploy(
|
||||
message: Optional[str] = typer.Option(None, "--message", "-m", help="Commit message"),
|
||||
no_push: bool = typer.Option(False, "--no-push", help="Don't push to remote"),
|
||||
):
|
||||
"""
|
||||
Deploy changes via Git.
|
||||
|
||||
Commits exported files and pushes to trigger Vercel deployment.
|
||||
"""
|
||||
setup_logging()
|
||||
deploy_command(message=message, push=not no_push)
|
||||
|
||||
|
||||
@app.command()
|
||||
def status():
|
||||
"""Show git status of the project."""
|
||||
setup_logging()
|
||||
status_command()
|
||||
|
||||
|
||||
# ==================== Utility Commands ====================
|
||||
|
||||
@app.command()
|
||||
def info():
|
||||
"""Show configuration and environment info."""
|
||||
setup_logging()
|
||||
|
||||
console.print("\n[bold blue]Odoo Directory CLI[/bold blue]\n")
|
||||
|
||||
try:
|
||||
settings = get_settings()
|
||||
|
||||
console.print("[bold]Configuration:[/bold]")
|
||||
console.print(f" Export directory: {settings.export_dir}")
|
||||
console.print(f" Images directory: {settings.images_dir}")
|
||||
console.print(f" Templates directory: {settings.templates_dir}")
|
||||
console.print(f" Log level: {settings.log_level}")
|
||||
console.print(f" Batch size: {settings.batch_size}")
|
||||
|
||||
console.print("\n[bold]API Status:[/bold]")
|
||||
console.print(f" Supabase URL: {settings.supabase_url[:30]}...")
|
||||
console.print(f" Supabase Key: {'*' * 10}... (configured)")
|
||||
console.print(f" Apify Token: {'*' * 10}... (configured)")
|
||||
console.print(f" Firecrawl Key: {'*' * 10}... (configured)")
|
||||
console.print(f" Anthropic Key: {'*' * 10}... (configured)")
|
||||
|
||||
except Exception as e:
|
||||
console.print(f"\n[red]Configuration error: {e}[/red]")
|
||||
console.print("\nMake sure you have a .env file with all required variables.")
|
||||
console.print("Copy .env.example to .env and fill in your API keys.")
|
||||
|
||||
|
||||
@app.command()
|
||||
def test_connection():
|
||||
"""Test connections to all external services."""
|
||||
setup_logging()
|
||||
|
||||
console.print("\n[bold blue]Testing Connections[/bold blue]\n")
|
||||
|
||||
# Test Supabase
|
||||
console.print("Testing Supabase...")
|
||||
try:
|
||||
from .services.supabase import SupabaseService
|
||||
db = SupabaseService()
|
||||
regions = db.list_regions()
|
||||
console.print(f" [green]Connected! Found {len(regions)} regions.[/green]")
|
||||
except Exception as e:
|
||||
console.print(f" [red]Failed: {e}[/red]")
|
||||
|
||||
# Test Anthropic
|
||||
console.print("Testing Anthropic...")
|
||||
try:
|
||||
from anthropic import Anthropic
|
||||
settings = get_settings()
|
||||
client = Anthropic(api_key=settings.anthropic_api_key)
|
||||
# Just verify the key format
|
||||
console.print(" [green]API key configured.[/green]")
|
||||
except Exception as e:
|
||||
console.print(f" [red]Failed: {e}[/red]")
|
||||
|
||||
# Note about other services
|
||||
console.print("\nNote: Apify and Firecrawl are tested during actual execution.")
|
||||
|
||||
|
||||
# ==================== Main Entry ====================
|
||||
|
||||
def main():
|
||||
"""Main entry point for the CLI."""
|
||||
app()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,15 @@
|
||||
"""CLI commands for Odoo Directory."""
|
||||
|
||||
from .setup import setup_command
|
||||
from .plan import plan_command
|
||||
from .execute import execute_command
|
||||
from .export import export_command
|
||||
from .deploy import deploy_command
|
||||
|
||||
__all__ = [
|
||||
"setup_command",
|
||||
"plan_command",
|
||||
"execute_command",
|
||||
"export_command",
|
||||
"deploy_command",
|
||||
]
|
||||
@@ -0,0 +1,148 @@
|
||||
"""
|
||||
Deploy command for pushing changes to Git.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from datetime import datetime
|
||||
from typing import Optional
|
||||
|
||||
import typer
|
||||
from rich.console import Console
|
||||
|
||||
from ..config import get_settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
console = Console()
|
||||
|
||||
|
||||
def deploy_command(
|
||||
message: Optional[str] = None,
|
||||
push: bool = True,
|
||||
) -> None:
|
||||
"""
|
||||
Deploy exported files via Git.
|
||||
|
||||
Adds all changes in the export directory, commits them,
|
||||
and optionally pushes to the remote repository.
|
||||
|
||||
Args:
|
||||
message: Custom commit message (auto-generated if not provided)
|
||||
push: Whether to push to remote after commit
|
||||
"""
|
||||
console.print("\n[bold blue]Deploying changes[/bold blue]\n")
|
||||
|
||||
settings = get_settings()
|
||||
project_root = Path(settings.export_dir).parent
|
||||
|
||||
# Verify we're in a git repo
|
||||
if not (project_root / ".git").exists():
|
||||
console.print("[red]Not a git repository![/red]")
|
||||
console.print(f"Expected .git directory in {project_root}")
|
||||
raise typer.Exit(1)
|
||||
|
||||
try:
|
||||
# Check for changes
|
||||
result = subprocess.run(
|
||||
["git", "status", "--porcelain"],
|
||||
cwd=project_root,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
)
|
||||
|
||||
if not result.stdout.strip():
|
||||
console.print("[yellow]No changes to deploy.[/yellow]")
|
||||
return
|
||||
|
||||
# Show changes
|
||||
console.print("[bold]Changes to deploy:[/bold]")
|
||||
console.print(result.stdout)
|
||||
|
||||
# Confirm deployment
|
||||
if not typer.confirm("Deploy these changes?"):
|
||||
console.print("[yellow]Deployment cancelled.[/yellow]")
|
||||
raise typer.Exit(0)
|
||||
|
||||
# Generate commit message if not provided
|
||||
if not message:
|
||||
timestamp = datetime.now().strftime("%Y-%m-%d %H:%M")
|
||||
message = f"Directory update: {timestamp}"
|
||||
|
||||
# Stage changes
|
||||
console.print("Staging changes...")
|
||||
subprocess.run(
|
||||
["git", "add", "export/", "static/images/companies/"],
|
||||
cwd=project_root,
|
||||
check=True,
|
||||
)
|
||||
|
||||
# Commit
|
||||
console.print(f"Committing: {message}")
|
||||
subprocess.run(
|
||||
["git", "commit", "-m", message],
|
||||
cwd=project_root,
|
||||
check=True,
|
||||
)
|
||||
|
||||
# Push if requested
|
||||
if push:
|
||||
console.print("Pushing to remote...")
|
||||
subprocess.run(
|
||||
["git", "push"],
|
||||
cwd=project_root,
|
||||
check=True,
|
||||
)
|
||||
console.print("[green]Changes pushed to remote.[/green]")
|
||||
else:
|
||||
console.print("[yellow]Changes committed locally but not pushed.[/yellow]")
|
||||
|
||||
console.print("\n[bold green]Deployment complete![/bold green]")
|
||||
|
||||
# Show deployment info
|
||||
console.print("\nVercel will automatically deploy the changes.")
|
||||
console.print("Check deployment status at: https://vercel.com/dashboard")
|
||||
|
||||
except subprocess.CalledProcessError as e:
|
||||
console.print(f"[red]Git command failed: {e}[/red]")
|
||||
if e.stderr:
|
||||
console.print(f"Error: {e.stderr}")
|
||||
raise typer.Exit(1)
|
||||
|
||||
except Exception as e:
|
||||
console.print(f"[red]Deployment failed: {e}[/red]")
|
||||
logger.exception("Deployment error")
|
||||
raise typer.Exit(1)
|
||||
|
||||
|
||||
def status_command() -> None:
|
||||
"""Show git status of the project."""
|
||||
settings = get_settings()
|
||||
project_root = Path(settings.export_dir).parent
|
||||
|
||||
try:
|
||||
# Git status
|
||||
result = subprocess.run(
|
||||
["git", "status"],
|
||||
cwd=project_root,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
)
|
||||
console.print(result.stdout)
|
||||
|
||||
# Last commit
|
||||
console.print("\n[bold]Last commit:[/bold]")
|
||||
result = subprocess.run(
|
||||
["git", "log", "-1", "--oneline"],
|
||||
cwd=project_root,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
)
|
||||
console.print(result.stdout)
|
||||
|
||||
except subprocess.CalledProcessError as e:
|
||||
console.print(f"[red]Git command failed: {e}[/red]")
|
||||
raise typer.Exit(1)
|
||||
@@ -0,0 +1,341 @@
|
||||
"""
|
||||
Execute command for running pipeline steps.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from typing import Optional, List
|
||||
from uuid import UUID
|
||||
import re
|
||||
|
||||
import typer
|
||||
from rich.console import Console
|
||||
from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn
|
||||
|
||||
from ..services.supabase import SupabaseService
|
||||
from ..services.apify import ApifyService
|
||||
from ..services.firecrawl import FirecrawlService
|
||||
from ..services.haiku import HaikuService
|
||||
from ..services.images import ImageService
|
||||
from ..services.tags import TagAggregationService
|
||||
from ..models.schemas import Company, StepStatus, TagCategory
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
console = Console()
|
||||
|
||||
|
||||
def execute_command(
|
||||
plan_id: str,
|
||||
step: Optional[str] = None,
|
||||
city: Optional[str] = None,
|
||||
dry_run: bool = False,
|
||||
) -> None:
|
||||
"""
|
||||
Execute an execution plan or specific step.
|
||||
|
||||
Runs the pipeline to scrape, enrich, and generate content for
|
||||
Odoo partners.
|
||||
|
||||
Args:
|
||||
plan_id: UUID of the execution plan
|
||||
step: Specific step to run (scrape, enrich, content, export)
|
||||
city: Specific city slug to process
|
||||
dry_run: If True, don't make actual API calls
|
||||
"""
|
||||
console.print(f"\n[bold blue]Executing Plan: {plan_id}[/bold blue]\n")
|
||||
|
||||
db = SupabaseService()
|
||||
|
||||
try:
|
||||
plan_uuid = UUID(plan_id)
|
||||
except ValueError:
|
||||
console.print(f"[red]Invalid plan ID: {plan_id}[/red]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
plan = db.get_execution_plan(plan_uuid)
|
||||
if not plan:
|
||||
console.print(f"[red]Plan not found: {plan_id}[/red]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
# Get country info
|
||||
country = db.get_country_by_code(plan.country_code)
|
||||
if not country:
|
||||
console.print(f"[red]Country not found: {plan.country_code}[/red]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
# Filter cities if specified
|
||||
city_slugs = plan.cities
|
||||
if city:
|
||||
if city not in city_slugs:
|
||||
console.print(f"[red]City {city} not in plan[/red]")
|
||||
raise typer.Exit(1)
|
||||
city_slugs = [city]
|
||||
|
||||
# Determine steps to run
|
||||
if step:
|
||||
steps_to_run = [step]
|
||||
else:
|
||||
steps_to_run = ["scrape", "enrich", "content"]
|
||||
|
||||
console.print(f"Country: {country.name}")
|
||||
console.print(f"Cities: {len(city_slugs)}")
|
||||
console.print(f"Steps: {', '.join(steps_to_run)}")
|
||||
console.print(f"Dry run: {dry_run}")
|
||||
console.print()
|
||||
|
||||
# Update plan status
|
||||
if not dry_run:
|
||||
db.update_execution_plan(
|
||||
plan_uuid, {"status": "running", "started_at": datetime.utcnow().isoformat()}
|
||||
)
|
||||
|
||||
# Run each step
|
||||
for step_name in steps_to_run:
|
||||
console.print(f"\n[bold cyan]Running step: {step_name}[/bold cyan]\n")
|
||||
|
||||
for city_slug in city_slugs:
|
||||
city_obj = db.get_city_by_slug(city_slug, country.id)
|
||||
if not city_obj:
|
||||
console.print(f"[yellow]City not found: {city_slug}[/yellow]")
|
||||
continue
|
||||
|
||||
try:
|
||||
if step_name == "scrape":
|
||||
_run_scrape_step(db, city_obj, country, dry_run)
|
||||
elif step_name == "enrich":
|
||||
_run_enrich_step(db, city_obj, dry_run)
|
||||
elif step_name == "content":
|
||||
_run_content_step(db, city_obj, country, dry_run)
|
||||
except Exception as e:
|
||||
console.print(f"[red]Error in {step_name} for {city_slug}: {e}[/red]")
|
||||
logger.exception(f"Step {step_name} failed for {city_slug}")
|
||||
|
||||
# Update plan status
|
||||
if not dry_run:
|
||||
db.update_execution_plan(
|
||||
plan_uuid, {"status": "completed", "completed_at": datetime.utcnow().isoformat()}
|
||||
)
|
||||
|
||||
console.print("\n[bold green]Execution complete![/bold green]")
|
||||
|
||||
|
||||
def _run_scrape_step(db: SupabaseService, city, country, dry_run: bool) -> None:
|
||||
"""Run the scrape step for a city."""
|
||||
console.print(f"Scraping {city.name}...")
|
||||
|
||||
if dry_run:
|
||||
console.print("[yellow]Dry run - skipping actual scrape[/yellow]")
|
||||
return
|
||||
|
||||
apify = ApifyService()
|
||||
|
||||
# Scrape Google Maps
|
||||
results = apify.scrape_city(
|
||||
city_name=city.name,
|
||||
language=country.language,
|
||||
country=country.code,
|
||||
)
|
||||
|
||||
# Deduplicate
|
||||
results = apify.deduplicate_results(results)
|
||||
console.print(f"Found {len(results)} unique companies")
|
||||
|
||||
# Save to database
|
||||
saved = 0
|
||||
for result in results:
|
||||
# Check if already exists
|
||||
if result.placeId:
|
||||
existing = db.get_company_by_place_id(result.placeId)
|
||||
if existing:
|
||||
continue
|
||||
|
||||
# Create slug from name
|
||||
slug = _create_slug(result.title)
|
||||
|
||||
# Extract location
|
||||
lat = lng = None
|
||||
if result.location:
|
||||
lat = result.location.get("lat")
|
||||
lng = result.location.get("lng")
|
||||
|
||||
company = Company(
|
||||
city_id=city.id,
|
||||
name=result.title,
|
||||
slug=slug,
|
||||
google_place_id=result.placeId,
|
||||
address=result.address,
|
||||
phone=result.phone,
|
||||
website=result.website,
|
||||
rating=result.rating,
|
||||
review_count=result.reviewsCount,
|
||||
latitude=lat,
|
||||
longitude=lng,
|
||||
status="scraped",
|
||||
last_scraped_at=datetime.utcnow(),
|
||||
)
|
||||
|
||||
db.create_company(company)
|
||||
saved += 1
|
||||
|
||||
console.print(f"[green]Saved {saved} new companies[/green]")
|
||||
|
||||
|
||||
def _run_enrich_step(db: SupabaseService, city, dry_run: bool) -> None:
|
||||
"""Run the enrich step for companies in a city."""
|
||||
console.print(f"Enriching companies in {city.name}...")
|
||||
|
||||
# Get companies that need enrichment
|
||||
companies = db.list_companies(city_id=city.id, status="scraped", limit=100)
|
||||
console.print(f"Found {len(companies)} companies to enrich")
|
||||
|
||||
if not companies or dry_run:
|
||||
if dry_run:
|
||||
console.print("[yellow]Dry run - skipping enrichment[/yellow]")
|
||||
return
|
||||
|
||||
firecrawl = FirecrawlService()
|
||||
|
||||
with Progress(
|
||||
SpinnerColumn(),
|
||||
TextColumn("[progress.description]{task.description}"),
|
||||
BarColumn(),
|
||||
TextColumn("[progress.percentage]{task.percentage:>3.0f}%"),
|
||||
console=console,
|
||||
) as progress:
|
||||
task = progress.add_task("Enriching...", total=len(companies))
|
||||
|
||||
for company in companies:
|
||||
if not company.website:
|
||||
progress.advance(task)
|
||||
continue
|
||||
|
||||
try:
|
||||
info = firecrawl.extract_company_info(company.website)
|
||||
|
||||
updates = {
|
||||
"website_markdown": info.get("markdown"),
|
||||
"logo_url": info.get("logo_url"),
|
||||
"email": info.get("email") or company.email,
|
||||
"status": "enriched",
|
||||
"last_enriched_at": datetime.utcnow().isoformat(),
|
||||
}
|
||||
|
||||
db.update_company(company.id, updates)
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to enrich {company.name}: {e}")
|
||||
|
||||
progress.advance(task)
|
||||
|
||||
console.print("[green]Enrichment complete[/green]")
|
||||
|
||||
|
||||
def _run_content_step(db: SupabaseService, city, country, dry_run: bool) -> None:
|
||||
"""Run the content generation step."""
|
||||
console.print(f"Generating content for {city.name}...")
|
||||
|
||||
# Get companies that need content
|
||||
companies = db.list_companies(city_id=city.id, status="enriched", limit=100)
|
||||
console.print(f"Found {len(companies)} companies for content generation")
|
||||
|
||||
if not companies or dry_run:
|
||||
if dry_run:
|
||||
console.print("[yellow]Dry run - skipping content generation[/yellow]")
|
||||
return
|
||||
|
||||
haiku = HaikuService()
|
||||
tags_service = TagAggregationService(db)
|
||||
|
||||
with Progress(
|
||||
SpinnerColumn(),
|
||||
TextColumn("[progress.description]{task.description}"),
|
||||
BarColumn(),
|
||||
TextColumn("[progress.percentage]{task.percentage:>3.0f}%"),
|
||||
console=console,
|
||||
) as progress:
|
||||
task = progress.add_task("Generating...", total=len(companies))
|
||||
|
||||
for company in companies:
|
||||
try:
|
||||
# Extract tags
|
||||
tags_response = haiku.extract_tags(
|
||||
company_name=company.name,
|
||||
website_markdown=company.website_markdown,
|
||||
address=company.address,
|
||||
)
|
||||
|
||||
# Save tags
|
||||
_save_company_tags(db, company.id, tags_response)
|
||||
|
||||
# Generate content
|
||||
content = haiku.generate_company_content(
|
||||
company_name=company.name,
|
||||
city_name=city.name,
|
||||
country_name=country.name,
|
||||
website_markdown=company.website_markdown,
|
||||
services=tags_response.services,
|
||||
industries=tags_response.industries,
|
||||
modules=tags_response.modules,
|
||||
partner_level=tags_response.partner_level,
|
||||
language=country.language,
|
||||
)
|
||||
|
||||
updates = {
|
||||
"description": content.description,
|
||||
"content_html": content.content_html,
|
||||
"meta_title": content.meta_title,
|
||||
"meta_description": content.meta_description,
|
||||
"status": "content_generated",
|
||||
"last_content_at": datetime.utcnow().isoformat(),
|
||||
}
|
||||
|
||||
db.update_company(company.id, updates)
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to generate content for {company.name}: {e}")
|
||||
|
||||
progress.advance(task)
|
||||
|
||||
# Aggregate tags for city
|
||||
console.print("Aggregating city tags...")
|
||||
tags_service.aggregate_city_tags(city.id)
|
||||
|
||||
console.print("[green]Content generation complete[/green]")
|
||||
|
||||
|
||||
def _save_company_tags(db: SupabaseService, company_id: UUID, tags_response) -> None:
|
||||
"""Save extracted tags to company."""
|
||||
# Clear existing tags
|
||||
db.remove_company_tags(company_id)
|
||||
|
||||
# Add service tags
|
||||
for name in tags_response.services:
|
||||
tag = db.get_or_create_tag(name, TagCategory.SERVICE)
|
||||
db.add_company_tag(company_id, tag.id, tags_response.confidence)
|
||||
|
||||
# Add industry tags
|
||||
for name in tags_response.industries:
|
||||
tag = db.get_or_create_tag(name, TagCategory.INDUSTRY)
|
||||
db.add_company_tag(company_id, tag.id, tags_response.confidence)
|
||||
|
||||
# Add module tags
|
||||
for name in tags_response.modules:
|
||||
tag = db.get_or_create_tag(name, TagCategory.MODULE)
|
||||
db.add_company_tag(company_id, tag.id, tags_response.confidence)
|
||||
|
||||
# Add partner level if present
|
||||
if tags_response.partner_level:
|
||||
tag = db.get_or_create_tag(tags_response.partner_level, TagCategory.PARTNER_LEVEL)
|
||||
db.add_company_tag(company_id, tag.id, tags_response.confidence)
|
||||
|
||||
|
||||
def _create_slug(name: str) -> str:
|
||||
"""Create URL-friendly slug from company name."""
|
||||
# Convert to lowercase
|
||||
slug = name.lower()
|
||||
# Replace spaces and special chars with hyphens
|
||||
slug = re.sub(r"[^a-z0-9]+", "-", slug)
|
||||
# Remove leading/trailing hyphens
|
||||
slug = slug.strip("-")
|
||||
# Limit length
|
||||
return slug[:100]
|
||||
@@ -0,0 +1,511 @@
|
||||
"""
|
||||
Export command for generating static files.
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import Optional, List, Dict, Any
|
||||
from uuid import UUID
|
||||
|
||||
import typer
|
||||
from rich.console import Console
|
||||
from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn
|
||||
from jinja2 import Environment, FileSystemLoader
|
||||
|
||||
from ..services.supabase import SupabaseService
|
||||
from ..services.tags import TagAggregationService
|
||||
from ..services.images import ImageService
|
||||
from ..config import get_settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
console = Console()
|
||||
|
||||
|
||||
def export_command(
|
||||
country: str,
|
||||
city: Optional[str] = None,
|
||||
output_dir: Optional[str] = None,
|
||||
) -> None:
|
||||
"""
|
||||
Export static XML/HTML files for a country or city.
|
||||
|
||||
Generates company pages, city hubs, country hubs, and sitemaps
|
||||
from the database content.
|
||||
|
||||
Args:
|
||||
country: Country code (e.g., DE)
|
||||
city: Optional specific city slug
|
||||
output_dir: Optional output directory override
|
||||
"""
|
||||
console.print(f"\n[bold blue]Exporting files for {country}[/bold blue]\n")
|
||||
|
||||
settings = get_settings()
|
||||
db = SupabaseService()
|
||||
tags_service = TagAggregationService(db)
|
||||
image_service = ImageService()
|
||||
|
||||
# Setup output directory
|
||||
if output_dir:
|
||||
export_path = Path(output_dir)
|
||||
else:
|
||||
export_path = settings.export_dir
|
||||
|
||||
export_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Get country info
|
||||
country_obj = db.get_country_by_code(country.upper())
|
||||
if not country_obj:
|
||||
console.print(f"[red]Country not found: {country}[/red]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
# Setup Jinja environment
|
||||
jinja_env = _setup_jinja_environment(settings.templates_dir)
|
||||
|
||||
# Get cities to export
|
||||
all_cities = db.list_cities(country_id=country_obj.id)
|
||||
|
||||
if city:
|
||||
cities_to_export = [c for c in all_cities if c.slug == city]
|
||||
if not cities_to_export:
|
||||
console.print(f"[red]City not found: {city}[/red]")
|
||||
raise typer.Exit(1)
|
||||
else:
|
||||
cities_to_export = all_cities
|
||||
|
||||
console.print(f"Exporting {len(cities_to_export)} cities...")
|
||||
|
||||
# Export each city
|
||||
total_companies = 0
|
||||
|
||||
with Progress(
|
||||
SpinnerColumn(),
|
||||
TextColumn("[progress.description]{task.description}"),
|
||||
BarColumn(),
|
||||
TextColumn("[progress.percentage]{task.percentage:>3.0f}%"),
|
||||
console=console,
|
||||
) as progress:
|
||||
task = progress.add_task("Exporting...", total=len(cities_to_export))
|
||||
|
||||
for city_obj in cities_to_export:
|
||||
city_companies = _export_city(
|
||||
db=db,
|
||||
tags_service=tags_service,
|
||||
image_service=image_service,
|
||||
jinja_env=jinja_env,
|
||||
country=country_obj,
|
||||
city=city_obj,
|
||||
export_path=export_path,
|
||||
)
|
||||
total_companies += city_companies
|
||||
progress.advance(task)
|
||||
|
||||
# Export country hub
|
||||
_export_country_hub(
|
||||
db=db,
|
||||
tags_service=tags_service,
|
||||
jinja_env=jinja_env,
|
||||
country=country_obj,
|
||||
cities=cities_to_export,
|
||||
export_path=export_path,
|
||||
)
|
||||
|
||||
# Export sitemaps
|
||||
_export_sitemaps(
|
||||
db=db,
|
||||
country=country_obj,
|
||||
cities=cities_to_export,
|
||||
export_path=export_path,
|
||||
)
|
||||
|
||||
# Export tag JSON files for frontend filtering
|
||||
_export_tag_json(
|
||||
db=db,
|
||||
tags_service=tags_service,
|
||||
country=country_obj,
|
||||
cities=cities_to_export,
|
||||
export_path=export_path,
|
||||
)
|
||||
|
||||
console.print(f"\n[bold green]Export complete![/bold green]")
|
||||
console.print(f" - Companies: {total_companies}")
|
||||
console.print(f" - Cities: {len(cities_to_export)}")
|
||||
console.print(f" - Output: {export_path}")
|
||||
|
||||
|
||||
def _setup_jinja_environment(templates_dir: Path) -> Environment:
|
||||
"""Setup Jinja2 templating environment."""
|
||||
# Create templates directory if it doesn't exist
|
||||
templates_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Create default templates if they don't exist
|
||||
_ensure_templates_exist(templates_dir)
|
||||
|
||||
return Environment(
|
||||
loader=FileSystemLoader(str(templates_dir)),
|
||||
autoescape=True,
|
||||
)
|
||||
|
||||
|
||||
def _ensure_templates_exist(templates_dir: Path) -> None:
|
||||
"""Create default templates if they don't exist."""
|
||||
# Company template
|
||||
company_template = templates_dir / "company.xml"
|
||||
if not company_template.exists():
|
||||
company_template.write_text(_DEFAULT_COMPANY_TEMPLATE)
|
||||
|
||||
# City hub template
|
||||
city_template = templates_dir / "city_hub.xml"
|
||||
if not city_template.exists():
|
||||
city_template.write_text(_DEFAULT_CITY_TEMPLATE)
|
||||
|
||||
# Country hub template
|
||||
country_template = templates_dir / "country_hub.xml"
|
||||
if not country_template.exists():
|
||||
country_template.write_text(_DEFAULT_COUNTRY_TEMPLATE)
|
||||
|
||||
|
||||
def _export_city(
|
||||
db: SupabaseService,
|
||||
tags_service: TagAggregationService,
|
||||
image_service: ImageService,
|
||||
jinja_env: Environment,
|
||||
country,
|
||||
city,
|
||||
export_path: Path,
|
||||
) -> int:
|
||||
"""Export all files for a city."""
|
||||
# Create city directory
|
||||
city_path = export_path / country.slug / city.slug
|
||||
city_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Get companies
|
||||
companies = db.list_companies(
|
||||
city_id=city.id,
|
||||
status="content_generated",
|
||||
limit=500,
|
||||
)
|
||||
|
||||
# Export each company
|
||||
company_template = jinja_env.get_template("company.xml")
|
||||
|
||||
for company in companies:
|
||||
company_tags = db.get_company_tags(company.id)
|
||||
|
||||
context = {
|
||||
"company": company,
|
||||
"city": city,
|
||||
"country": country,
|
||||
"tags": company_tags,
|
||||
"services": [t for t in company_tags if hasattr(t, "category") and t.category == "service"],
|
||||
"industries": [t for t in company_tags if hasattr(t, "category") and t.category == "industry"],
|
||||
"modules": [t for t in company_tags if hasattr(t, "category") and t.category == "module"],
|
||||
}
|
||||
|
||||
content = company_template.render(**context)
|
||||
company_file = city_path / f"{company.slug}.html"
|
||||
company_file.write_text(content, encoding="utf-8")
|
||||
|
||||
# Export city hub
|
||||
city_tags = tags_service.get_city_tags_by_category(city.id)
|
||||
hub_template = jinja_env.get_template("city_hub.xml")
|
||||
|
||||
hub_context = {
|
||||
"city": city,
|
||||
"country": country,
|
||||
"companies": companies,
|
||||
"company_count": len(companies),
|
||||
"tags": city_tags,
|
||||
}
|
||||
|
||||
hub_content = hub_template.render(**hub_context)
|
||||
hub_file = city_path / "index.html"
|
||||
hub_file.write_text(hub_content, encoding="utf-8")
|
||||
|
||||
return len(companies)
|
||||
|
||||
|
||||
def _export_country_hub(
|
||||
db: SupabaseService,
|
||||
tags_service: TagAggregationService,
|
||||
jinja_env: Environment,
|
||||
country,
|
||||
cities: List,
|
||||
export_path: Path,
|
||||
) -> None:
|
||||
"""Export country hub page."""
|
||||
country_path = export_path / country.slug
|
||||
country_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Get aggregate data
|
||||
total_companies = 0
|
||||
city_data = []
|
||||
|
||||
for city in cities:
|
||||
count = db.count_companies(city_id=city.id, status="content_generated")
|
||||
total_companies += count
|
||||
city_data.append({
|
||||
"city": city,
|
||||
"company_count": count,
|
||||
})
|
||||
|
||||
# Sort cities by company count
|
||||
city_data.sort(key=lambda x: x["company_count"], reverse=True)
|
||||
|
||||
country_tags = tags_service.get_country_tags_by_category(country.id)
|
||||
|
||||
# Render template
|
||||
template = jinja_env.get_template("country_hub.xml")
|
||||
content = template.render(
|
||||
country=country,
|
||||
cities=city_data,
|
||||
city_count=len(cities),
|
||||
company_count=total_companies,
|
||||
tags=country_tags,
|
||||
)
|
||||
|
||||
hub_file = country_path / "index.html"
|
||||
hub_file.write_text(content, encoding="utf-8")
|
||||
|
||||
|
||||
def _export_sitemaps(
|
||||
db: SupabaseService,
|
||||
country,
|
||||
cities: List,
|
||||
export_path: Path,
|
||||
) -> None:
|
||||
"""Export XML sitemaps."""
|
||||
sitemap_path = export_path / "sitemaps"
|
||||
sitemap_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
base_url = "https://odoo-expertos.com"
|
||||
|
||||
# Company sitemap
|
||||
urls = []
|
||||
for city in cities:
|
||||
companies = db.list_companies(
|
||||
city_id=city.id,
|
||||
status="content_generated",
|
||||
limit=500,
|
||||
)
|
||||
for company in companies:
|
||||
urls.append(
|
||||
f"{base_url}/{country.slug}/{city.slug}/{company.slug}.html"
|
||||
)
|
||||
|
||||
sitemap_xml = _generate_sitemap_xml(urls)
|
||||
sitemap_file = sitemap_path / f"sitemap-companies-{country.code.lower()}.xml"
|
||||
sitemap_file.write_text(sitemap_xml, encoding="utf-8")
|
||||
|
||||
# City sitemap
|
||||
city_urls = [
|
||||
f"{base_url}/{country.slug}/{city.slug}/"
|
||||
for city in cities
|
||||
]
|
||||
city_sitemap = _generate_sitemap_xml(city_urls)
|
||||
city_sitemap_file = sitemap_path / f"sitemap-cities-{country.code.lower()}.xml"
|
||||
city_sitemap_file.write_text(city_sitemap, encoding="utf-8")
|
||||
|
||||
|
||||
def _generate_sitemap_xml(urls: List[str]) -> str:
|
||||
"""Generate sitemap XML content."""
|
||||
lines = ['<?xml version="1.0" encoding="UTF-8"?>']
|
||||
lines.append('<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">')
|
||||
|
||||
for url in urls:
|
||||
lines.append(" <url>")
|
||||
lines.append(f" <loc>{url}</loc>")
|
||||
lines.append(" <changefreq>monthly</changefreq>")
|
||||
lines.append(" <priority>0.7</priority>")
|
||||
lines.append(" </url>")
|
||||
|
||||
lines.append("</urlset>")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def _export_tag_json(
|
||||
db: SupabaseService,
|
||||
tags_service: TagAggregationService,
|
||||
country,
|
||||
cities: List,
|
||||
export_path: Path,
|
||||
) -> None:
|
||||
"""Export tag aggregations as JSON for frontend filtering."""
|
||||
json_path = export_path / "tags"
|
||||
json_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# City tags
|
||||
cities_path = json_path / "cities"
|
||||
cities_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
for city in cities:
|
||||
city_json = tags_service.export_city_tags_json(city.id)
|
||||
json_file = cities_path / f"{city.slug}.json"
|
||||
json_file.write_text(json.dumps(city_json, indent=2), encoding="utf-8")
|
||||
|
||||
# Country tags
|
||||
country_json = tags_service.export_country_tags_json(country.id)
|
||||
country_file = json_path / f"{country.slug}.json"
|
||||
country_file.write_text(json.dumps(country_json, indent=2), encoding="utf-8")
|
||||
|
||||
|
||||
# Default templates
|
||||
|
||||
_DEFAULT_COMPANY_TEMPLATE = """<!DOCTYPE html>
|
||||
<html lang="{{ country.language }}">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>{{ company.meta_title or company.name }}</title>
|
||||
<meta name="description" content="{{ company.meta_description or company.description }}">
|
||||
<link rel="canonical" href="https://odoo-expertos.com/{{ country.slug }}/{{ city.slug }}/{{ company.slug }}.html">
|
||||
</head>
|
||||
<body>
|
||||
<header>
|
||||
<nav>
|
||||
<a href="/{{ country.slug }}/">{{ country.name }}</a> >
|
||||
<a href="/{{ country.slug }}/{{ city.slug }}/">{{ city.name }}</a> >
|
||||
{{ company.name }}
|
||||
</nav>
|
||||
</header>
|
||||
|
||||
<main>
|
||||
<article>
|
||||
<h1>{{ company.name }}</h1>
|
||||
|
||||
{% if company.description %}
|
||||
<p class="lead">{{ company.description }}</p>
|
||||
{% endif %}
|
||||
|
||||
<div class="company-info">
|
||||
{% if company.address %}
|
||||
<p><strong>Address:</strong> {{ company.address }}</p>
|
||||
{% endif %}
|
||||
{% if company.phone %}
|
||||
<p><strong>Phone:</strong> {{ company.phone }}</p>
|
||||
{% endif %}
|
||||
{% if company.website %}
|
||||
<p><strong>Website:</strong> <a href="{{ company.website }}" rel="noopener">{{ company.website }}</a></p>
|
||||
{% endif %}
|
||||
{% if company.email %}
|
||||
<p><strong>Email:</strong> {{ company.email }}</p>
|
||||
{% endif %}
|
||||
{% if company.rating %}
|
||||
<p><strong>Rating:</strong> {{ company.rating }}/5 ({{ company.review_count }} reviews)</p>
|
||||
{% endif %}
|
||||
</div>
|
||||
|
||||
<div class="content">
|
||||
{{ company.content_html | safe }}
|
||||
</div>
|
||||
</article>
|
||||
</main>
|
||||
|
||||
<footer>
|
||||
<p>© Odoo Expertos - {{ city.name }}, {{ country.name }}</p>
|
||||
</footer>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
|
||||
_DEFAULT_CITY_TEMPLATE = """<!DOCTYPE html>
|
||||
<html lang="{{ country.language }}">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>Odoo Partners in {{ city.name }} | {{ country.name }}</title>
|
||||
<meta name="description" content="Find {{ company_count }} certified Odoo partners in {{ city.name }}, {{ country.name }}. Compare services, expertise, and reviews.">
|
||||
<link rel="canonical" href="https://odoo-expertos.com/{{ country.slug }}/{{ city.slug }}/">
|
||||
</head>
|
||||
<body>
|
||||
<header>
|
||||
<nav>
|
||||
<a href="/{{ country.slug }}/">{{ country.name }}</a> >
|
||||
{{ city.name }}
|
||||
</nav>
|
||||
</header>
|
||||
|
||||
<main>
|
||||
<h1>Odoo Partners in {{ city.name }}</h1>
|
||||
<p>Discover {{ company_count }} certified Odoo partners in {{ city.name }}, {{ country.name }}.</p>
|
||||
|
||||
<section class="filters">
|
||||
<h2>Filter by Services</h2>
|
||||
<ul>
|
||||
{% for tag in tags.services %}
|
||||
<li>{{ tag.name }} ({{ tag.count }})</li>
|
||||
{% endfor %}
|
||||
</ul>
|
||||
</section>
|
||||
|
||||
<section class="companies">
|
||||
<h2>Odoo Partners</h2>
|
||||
{% for company in companies %}
|
||||
<article class="company-card">
|
||||
<h3><a href="{{ company.slug }}.html">{{ company.name }}</a></h3>
|
||||
{% if company.description %}
|
||||
<p>{{ company.description }}</p>
|
||||
{% endif %}
|
||||
{% if company.rating %}
|
||||
<p class="rating">{{ company.rating }}/5</p>
|
||||
{% endif %}
|
||||
</article>
|
||||
{% endfor %}
|
||||
</section>
|
||||
</main>
|
||||
|
||||
<footer>
|
||||
<p>© Odoo Expertos - {{ city.name }}, {{ country.name }}</p>
|
||||
</footer>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
|
||||
_DEFAULT_COUNTRY_TEMPLATE = """<!DOCTYPE html>
|
||||
<html lang="{{ country.language }}">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>Odoo Partners in {{ country.name }}</title>
|
||||
<meta name="description" content="Find {{ company_count }} certified Odoo partners in {{ city_count }} cities across {{ country.name }}.">
|
||||
<link rel="canonical" href="https://odoo-expertos.com/{{ country.slug }}/">
|
||||
</head>
|
||||
<body>
|
||||
<header>
|
||||
<nav>
|
||||
{{ country.name }}
|
||||
</nav>
|
||||
</header>
|
||||
|
||||
<main>
|
||||
<h1>Odoo Partners in {{ country.name }}</h1>
|
||||
<p>Discover {{ company_count }} certified Odoo partners in {{ city_count }} cities across {{ country.name }}.</p>
|
||||
|
||||
<section class="cities">
|
||||
<h2>Cities</h2>
|
||||
<ul>
|
||||
{% for item in cities %}
|
||||
<li>
|
||||
<a href="{{ item.city.slug }}/">{{ item.city.name }}</a>
|
||||
({{ item.company_count }} partners)
|
||||
</li>
|
||||
{% endfor %}
|
||||
</ul>
|
||||
</section>
|
||||
|
||||
<section class="filters">
|
||||
<h2>Services</h2>
|
||||
<ul>
|
||||
{% for tag in tags.services %}
|
||||
<li>{{ tag.name }} ({{ tag.company_count }} companies in {{ tag.city_count }} cities)</li>
|
||||
{% endfor %}
|
||||
</ul>
|
||||
</section>
|
||||
</main>
|
||||
|
||||
<footer>
|
||||
<p>© Odoo Expertos - {{ country.name }}</p>
|
||||
</footer>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
@@ -0,0 +1,242 @@
|
||||
"""
|
||||
Plan command for creating execution plans.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from typing import List, Optional
|
||||
from uuid import UUID
|
||||
|
||||
import typer
|
||||
from rich.console import Console
|
||||
from rich.table import Table
|
||||
|
||||
from ..services.supabase import SupabaseService
|
||||
from ..models.schemas import ExecutionPlan, PlanStep, StepStatus
|
||||
from ..config import SEARCH_QUERIES
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
console = Console()
|
||||
|
||||
|
||||
def plan_command(
|
||||
country: str,
|
||||
cities: Optional[List[str]] = None,
|
||||
name: Optional[str] = None,
|
||||
) -> Optional[UUID]:
|
||||
"""
|
||||
Create an execution plan for scraping Odoo partners.
|
||||
|
||||
This generates a plan that specifies which cities to scrape and
|
||||
what steps to perform. The plan can then be executed with the
|
||||
execute command.
|
||||
|
||||
Args:
|
||||
country: Country code (e.g., DE for Germany)
|
||||
cities: List of city slugs to include (default: all cities in country)
|
||||
name: Optional name for the plan
|
||||
|
||||
Returns:
|
||||
Plan UUID if successful
|
||||
"""
|
||||
console.print(f"\n[bold blue]Creating Execution Plan for {country}[/bold blue]\n")
|
||||
|
||||
db = SupabaseService()
|
||||
|
||||
# Verify country exists
|
||||
country_obj = db.get_country_by_code(country.upper())
|
||||
if not country_obj:
|
||||
console.print(f"[red]Country not found: {country}[/red]")
|
||||
console.print("Run 'setup' command first to seed country data.")
|
||||
raise typer.Exit(1)
|
||||
|
||||
# Get cities
|
||||
all_cities = db.list_cities(country_id=country_obj.id)
|
||||
|
||||
if cities:
|
||||
# Filter to specified cities
|
||||
city_slugs = [c.lower() for c in cities]
|
||||
selected_cities = [c for c in all_cities if c.slug in city_slugs]
|
||||
|
||||
missing = set(city_slugs) - {c.slug for c in selected_cities}
|
||||
if missing:
|
||||
console.print(f"[yellow]Warning: Cities not found: {missing}[/yellow]")
|
||||
else:
|
||||
selected_cities = all_cities
|
||||
|
||||
if not selected_cities:
|
||||
console.print("[red]No cities found for this country.[/red]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
# Generate plan name
|
||||
if not name:
|
||||
city_names = [c.slug for c in selected_cities[:3]]
|
||||
name = f"{country.upper()}-{'-'.join(city_names)}"
|
||||
if len(selected_cities) > 3:
|
||||
name += f"-and-{len(selected_cities) - 3}-more"
|
||||
|
||||
# Estimate company counts based on queries
|
||||
language = country_obj.language
|
||||
queries_per_city = len(SEARCH_QUERIES.get(language, SEARCH_QUERIES["en"]))
|
||||
estimated_companies = len(selected_cities) * queries_per_city * 15 # ~15 per query
|
||||
|
||||
# Show plan summary
|
||||
table = Table(title="Execution Plan Summary")
|
||||
table.add_column("Field", style="cyan")
|
||||
table.add_column("Value", style="green")
|
||||
|
||||
table.add_row("Plan Name", name)
|
||||
table.add_row("Country", f"{country_obj.name} ({country_obj.code})")
|
||||
table.add_row("Language", country_obj.language)
|
||||
table.add_row("Cities", str(len(selected_cities)))
|
||||
table.add_row("Queries/City", str(queries_per_city))
|
||||
table.add_row("Est. Companies", f"~{estimated_companies}")
|
||||
|
||||
console.print(table)
|
||||
console.print()
|
||||
|
||||
# Show city list
|
||||
console.print("[bold]Cities included:[/bold]")
|
||||
for city in selected_cities:
|
||||
console.print(f" - {city.name} ({city.slug})")
|
||||
|
||||
console.print()
|
||||
|
||||
# Confirm creation
|
||||
if not typer.confirm("Create this plan?"):
|
||||
console.print("[yellow]Plan creation cancelled.[/yellow]")
|
||||
raise typer.Exit(0)
|
||||
|
||||
# Create the plan
|
||||
plan = ExecutionPlan(
|
||||
name=name,
|
||||
country_code=country.upper(),
|
||||
cities=[c.slug for c in selected_cities],
|
||||
status=StepStatus.PENDING,
|
||||
)
|
||||
|
||||
plan = db.create_execution_plan(plan)
|
||||
console.print(f"\n[green]Plan created: {plan.id}[/green]")
|
||||
|
||||
# Create steps for each city
|
||||
step_types = ["scrape", "enrich", "content", "export"]
|
||||
|
||||
for city in selected_cities:
|
||||
for step_type in step_types:
|
||||
step = PlanStep(
|
||||
plan_id=plan.id,
|
||||
step_name=step_type,
|
||||
city_id=city.id,
|
||||
status=StepStatus.PENDING,
|
||||
)
|
||||
db.create_plan_step(step)
|
||||
|
||||
console.print(
|
||||
f"[green]Created {len(selected_cities) * len(step_types)} steps[/green]"
|
||||
)
|
||||
|
||||
# Print next steps
|
||||
console.print("\n[bold]Next steps:[/bold]")
|
||||
console.print(f" 1. Run: python -m directory.cli execute --plan {plan.id}")
|
||||
console.print(" 2. Or run step by step:")
|
||||
console.print(f" python -m directory.cli execute --plan {plan.id} --step scrape")
|
||||
console.print(
|
||||
f" python -m directory.cli execute --plan {plan.id} --step enrich"
|
||||
)
|
||||
console.print(
|
||||
f" python -m directory.cli execute --plan {plan.id} --step content"
|
||||
)
|
||||
console.print()
|
||||
|
||||
return plan.id
|
||||
|
||||
|
||||
def list_plans() -> None:
|
||||
"""List all execution plans."""
|
||||
db = SupabaseService()
|
||||
plans = db.list_execution_plans()
|
||||
|
||||
if not plans:
|
||||
console.print("[yellow]No execution plans found.[/yellow]")
|
||||
return
|
||||
|
||||
table = Table(title="Execution Plans")
|
||||
table.add_column("ID", style="cyan", max_width=36)
|
||||
table.add_column("Name", style="green")
|
||||
table.add_column("Country", style="blue")
|
||||
table.add_column("Cities", style="magenta")
|
||||
table.add_column("Status", style="yellow")
|
||||
table.add_column("Created", style="dim")
|
||||
|
||||
for plan in plans:
|
||||
table.add_row(
|
||||
str(plan.id),
|
||||
plan.name,
|
||||
plan.country_code,
|
||||
str(len(plan.cities)),
|
||||
plan.status.value if hasattr(plan.status, "value") else plan.status,
|
||||
str(plan.created_at)[:16] if plan.created_at else "-",
|
||||
)
|
||||
|
||||
console.print(table)
|
||||
|
||||
|
||||
def show_plan(plan_id: str) -> None:
|
||||
"""Show details of a specific plan."""
|
||||
db = SupabaseService()
|
||||
|
||||
try:
|
||||
plan_uuid = UUID(plan_id)
|
||||
except ValueError:
|
||||
console.print(f"[red]Invalid plan ID: {plan_id}[/red]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
plan = db.get_execution_plan(plan_uuid)
|
||||
|
||||
if not plan:
|
||||
console.print(f"[red]Plan not found: {plan_id}[/red]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
# Plan info
|
||||
console.print(f"\n[bold blue]Plan: {plan.name}[/bold blue]\n")
|
||||
|
||||
table = Table()
|
||||
table.add_column("Field", style="cyan")
|
||||
table.add_column("Value", style="green")
|
||||
|
||||
table.add_row("ID", str(plan.id))
|
||||
table.add_row("Country", plan.country_code)
|
||||
table.add_row("Cities", ", ".join(plan.cities))
|
||||
table.add_row("Status", plan.status.value if hasattr(plan.status, "value") else plan.status)
|
||||
table.add_row("Created", str(plan.created_at)[:19] if plan.created_at else "-")
|
||||
table.add_row("Started", str(plan.started_at)[:19] if plan.started_at else "-")
|
||||
table.add_row("Completed", str(plan.completed_at)[:19] if plan.completed_at else "-")
|
||||
|
||||
console.print(table)
|
||||
|
||||
# Steps
|
||||
if plan.steps:
|
||||
console.print("\n[bold]Steps:[/bold]")
|
||||
|
||||
steps_table = Table()
|
||||
steps_table.add_column("Step", style="cyan")
|
||||
steps_table.add_column("City", style="blue")
|
||||
steps_table.add_column("Status", style="yellow")
|
||||
steps_table.add_column("Progress", style="green")
|
||||
|
||||
for step in plan.steps:
|
||||
city_slug = "-"
|
||||
if step.city_id:
|
||||
city = db.get_city_by_slug("", step.city_id)
|
||||
# We don't have a get_city_by_id, so we'll just show the ID
|
||||
city_slug = str(step.city_id)[:8]
|
||||
|
||||
progress = f"{step.items_processed}/{step.items_total}" if step.items_total else "-"
|
||||
|
||||
steps_table.add_row(
|
||||
step.step_name,
|
||||
city_slug,
|
||||
step.status.value if hasattr(step.status, "value") else step.status,
|
||||
progress,
|
||||
)
|
||||
|
||||
console.print(steps_table)
|
||||
@@ -0,0 +1,304 @@
|
||||
"""
|
||||
Setup command for initializing the database.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from typing import Optional
|
||||
|
||||
import typer
|
||||
from rich.console import Console
|
||||
from rich.progress import Progress, SpinnerColumn, TextColumn
|
||||
|
||||
from ..services.supabase import SupabaseService
|
||||
from ..models.schemas import Region, Country, City, Tag, TagCategory
|
||||
from ..config import DEFAULT_TAGS, GERMANY_CITIES
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
console = Console()
|
||||
|
||||
|
||||
# SQL schema for creating tables
|
||||
SCHEMA_SQL = """
|
||||
-- Enable UUID extension
|
||||
CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
|
||||
|
||||
-- Regions table
|
||||
CREATE TABLE IF NOT EXISTS regions (
|
||||
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||
name TEXT NOT NULL,
|
||||
slug TEXT NOT NULL UNIQUE,
|
||||
created_at TIMESTAMPTZ DEFAULT NOW()
|
||||
);
|
||||
|
||||
-- Countries table
|
||||
CREATE TABLE IF NOT EXISTS countries (
|
||||
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||
region_id UUID REFERENCES regions(id),
|
||||
name TEXT NOT NULL,
|
||||
name_local TEXT,
|
||||
code TEXT NOT NULL UNIQUE,
|
||||
slug TEXT NOT NULL UNIQUE,
|
||||
language TEXT NOT NULL,
|
||||
created_at TIMESTAMPTZ DEFAULT NOW()
|
||||
);
|
||||
|
||||
-- Cities table
|
||||
CREATE TABLE IF NOT EXISTS cities (
|
||||
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||
country_id UUID REFERENCES countries(id),
|
||||
name TEXT NOT NULL,
|
||||
name_local TEXT,
|
||||
slug TEXT NOT NULL,
|
||||
state TEXT,
|
||||
population INTEGER,
|
||||
latitude FLOAT,
|
||||
longitude FLOAT,
|
||||
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
UNIQUE(country_id, slug)
|
||||
);
|
||||
|
||||
-- Tags table
|
||||
CREATE TABLE IF NOT EXISTS tags (
|
||||
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||
category TEXT NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
slug TEXT NOT NULL UNIQUE,
|
||||
description TEXT,
|
||||
created_at TIMESTAMPTZ DEFAULT NOW()
|
||||
);
|
||||
|
||||
-- Companies table
|
||||
CREATE TABLE IF NOT EXISTS companies (
|
||||
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||
city_id UUID REFERENCES cities(id),
|
||||
name TEXT NOT NULL,
|
||||
slug TEXT NOT NULL,
|
||||
google_place_id TEXT UNIQUE,
|
||||
address TEXT,
|
||||
phone TEXT,
|
||||
website TEXT,
|
||||
email TEXT,
|
||||
rating FLOAT,
|
||||
review_count INTEGER,
|
||||
latitude FLOAT,
|
||||
longitude FLOAT,
|
||||
website_markdown TEXT,
|
||||
logo_url TEXT,
|
||||
description TEXT,
|
||||
content_html TEXT,
|
||||
meta_title TEXT,
|
||||
meta_description TEXT,
|
||||
status TEXT DEFAULT 'scraped',
|
||||
last_scraped_at TIMESTAMPTZ,
|
||||
last_enriched_at TIMESTAMPTZ,
|
||||
last_content_at TIMESTAMPTZ,
|
||||
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ DEFAULT NOW()
|
||||
);
|
||||
|
||||
-- Company tags junction table
|
||||
CREATE TABLE IF NOT EXISTS company_tags (
|
||||
company_id UUID REFERENCES companies(id) ON DELETE CASCADE,
|
||||
tag_id UUID REFERENCES tags(id) ON DELETE CASCADE,
|
||||
confidence FLOAT DEFAULT 1.0,
|
||||
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
PRIMARY KEY (company_id, tag_id)
|
||||
);
|
||||
|
||||
-- City tag aggregation
|
||||
CREATE TABLE IF NOT EXISTS city_tags (
|
||||
city_id UUID REFERENCES cities(id) ON DELETE CASCADE,
|
||||
tag_id UUID REFERENCES tags(id) ON DELETE CASCADE,
|
||||
company_count INTEGER DEFAULT 0,
|
||||
PRIMARY KEY (city_id, tag_id)
|
||||
);
|
||||
|
||||
-- Country tag aggregation
|
||||
CREATE TABLE IF NOT EXISTS country_tags (
|
||||
country_id UUID REFERENCES countries(id) ON DELETE CASCADE,
|
||||
tag_id UUID REFERENCES tags(id) ON DELETE CASCADE,
|
||||
company_count INTEGER DEFAULT 0,
|
||||
city_count INTEGER DEFAULT 0,
|
||||
PRIMARY KEY (country_id, tag_id)
|
||||
);
|
||||
|
||||
-- Execution plans
|
||||
CREATE TABLE IF NOT EXISTS execution_plans (
|
||||
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||
name TEXT NOT NULL,
|
||||
country_code TEXT NOT NULL,
|
||||
cities TEXT[] NOT NULL,
|
||||
status TEXT DEFAULT 'pending',
|
||||
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
started_at TIMESTAMPTZ,
|
||||
completed_at TIMESTAMPTZ
|
||||
);
|
||||
|
||||
-- Plan steps
|
||||
CREATE TABLE IF NOT EXISTS plan_steps (
|
||||
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||
plan_id UUID REFERENCES execution_plans(id) ON DELETE CASCADE,
|
||||
step_name TEXT NOT NULL,
|
||||
city_id UUID REFERENCES cities(id),
|
||||
status TEXT DEFAULT 'pending',
|
||||
started_at TIMESTAMPTZ,
|
||||
completed_at TIMESTAMPTZ,
|
||||
error_message TEXT,
|
||||
items_processed INTEGER DEFAULT 0,
|
||||
items_total INTEGER DEFAULT 0
|
||||
);
|
||||
|
||||
-- Error logs
|
||||
CREATE TABLE IF NOT EXISTS error_logs (
|
||||
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||
entity_type TEXT NOT NULL,
|
||||
entity_id UUID,
|
||||
operation TEXT NOT NULL,
|
||||
error_message TEXT NOT NULL,
|
||||
details JSONB DEFAULT '{}',
|
||||
created_at TIMESTAMPTZ DEFAULT NOW()
|
||||
);
|
||||
|
||||
-- Indexes for performance
|
||||
CREATE INDEX IF NOT EXISTS idx_companies_city ON companies(city_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_companies_status ON companies(status);
|
||||
CREATE INDEX IF NOT EXISTS idx_company_tags_company ON company_tags(company_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_company_tags_tag ON company_tags(tag_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_city_tags_city ON city_tags(city_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_country_tags_country ON country_tags(country_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_plan_steps_plan ON plan_steps(plan_id);
|
||||
"""
|
||||
|
||||
|
||||
def setup_command(
|
||||
seed_tags: bool = True,
|
||||
seed_germany: bool = True,
|
||||
) -> None:
|
||||
"""
|
||||
Initialize the database with schema and seed data.
|
||||
|
||||
This command should be run once when setting up a new installation.
|
||||
It creates all required tables and optionally seeds reference data.
|
||||
|
||||
Args:
|
||||
seed_tags: Whether to seed default tags
|
||||
seed_germany: Whether to seed Germany cities
|
||||
"""
|
||||
console.print("\n[bold blue]Odoo Directory Setup[/bold blue]\n")
|
||||
|
||||
db = SupabaseService()
|
||||
|
||||
with Progress(
|
||||
SpinnerColumn(),
|
||||
TextColumn("[progress.description]{task.description}"),
|
||||
console=console,
|
||||
) as progress:
|
||||
# Note: Schema should be created in Supabase SQL Editor
|
||||
# The CLI just verifies connectivity and seeds data
|
||||
|
||||
task = progress.add_task("Verifying database connection...", total=None)
|
||||
try:
|
||||
# Test connection by listing regions
|
||||
regions = db.list_regions()
|
||||
progress.update(task, description="[green]Database connected")
|
||||
except Exception as e:
|
||||
console.print(f"\n[red]Database connection failed: {e}[/red]")
|
||||
console.print(
|
||||
"\n[yellow]Please create tables using the SQL schema in Supabase SQL Editor.[/yellow]"
|
||||
)
|
||||
console.print("\nSchema SQL can be found in directory/commands/setup.py")
|
||||
raise typer.Exit(1)
|
||||
|
||||
# Seed tags
|
||||
if seed_tags:
|
||||
task = progress.add_task("Seeding tags...", total=None)
|
||||
tag_count = _seed_tags(db)
|
||||
progress.update(task, description=f"[green]Seeded {tag_count} tags")
|
||||
|
||||
# Seed Germany
|
||||
if seed_germany:
|
||||
task = progress.add_task("Seeding Germany data...", total=None)
|
||||
city_count = _seed_germany(db)
|
||||
progress.update(task, description=f"[green]Seeded {city_count} cities")
|
||||
|
||||
console.print("\n[bold green]Setup complete![/bold green]\n")
|
||||
|
||||
|
||||
def _seed_tags(db: SupabaseService) -> int:
|
||||
"""Seed default tags into database."""
|
||||
count = 0
|
||||
|
||||
# Map category names to enum values
|
||||
category_map = {
|
||||
"services": TagCategory.SERVICE,
|
||||
"industries": TagCategory.INDUSTRY,
|
||||
"modules": TagCategory.MODULE,
|
||||
"partner_levels": TagCategory.PARTNER_LEVEL,
|
||||
}
|
||||
|
||||
for category_name, tag_names in DEFAULT_TAGS.items():
|
||||
category = category_map[category_name]
|
||||
|
||||
for name in tag_names:
|
||||
slug = name.lower().replace(" ", "-").replace("&", "and")
|
||||
existing = db.get_tag_by_slug(slug)
|
||||
|
||||
if not existing:
|
||||
db.create_tag(
|
||||
Tag(
|
||||
category=category,
|
||||
name=name,
|
||||
slug=slug,
|
||||
)
|
||||
)
|
||||
count += 1
|
||||
|
||||
return count
|
||||
|
||||
|
||||
def _seed_germany(db: SupabaseService) -> int:
|
||||
"""Seed Germany region, country, and cities."""
|
||||
# Create Europe region if not exists
|
||||
region = db.get_region_by_slug("europe")
|
||||
if not region:
|
||||
region = db.create_region(Region(name="Europe", slug="europe"))
|
||||
|
||||
# Create Germany if not exists
|
||||
country = db.get_country_by_code("DE")
|
||||
if not country:
|
||||
country = db.create_country(
|
||||
Country(
|
||||
region_id=region.id,
|
||||
name="Germany",
|
||||
name_local="Deutschland",
|
||||
code="DE",
|
||||
slug="deutschland",
|
||||
language="de",
|
||||
)
|
||||
)
|
||||
|
||||
# Create cities
|
||||
count = 0
|
||||
for city_data in GERMANY_CITIES:
|
||||
existing = db.get_city_by_slug(city_data["slug"], country.id)
|
||||
if not existing:
|
||||
db.create_city(
|
||||
City(
|
||||
country_id=country.id,
|
||||
name=city_data["name"],
|
||||
slug=city_data["slug"],
|
||||
state=city_data.get("state"),
|
||||
)
|
||||
)
|
||||
count += 1
|
||||
|
||||
return count
|
||||
|
||||
|
||||
def print_schema() -> None:
|
||||
"""Print the SQL schema for manual creation."""
|
||||
console.print("\n[bold]Database Schema SQL:[/bold]\n")
|
||||
console.print(SCHEMA_SQL)
|
||||
console.print(
|
||||
"\n[yellow]Copy this SQL and run it in your Supabase SQL Editor.[/yellow]"
|
||||
)
|
||||
@@ -0,0 +1,161 @@
|
||||
"""
|
||||
Configuration management for Odoo Directory CLI.
|
||||
Loads environment variables and provides typed configuration.
|
||||
"""
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
from pydantic_settings import BaseSettings
|
||||
from pydantic import Field
|
||||
from dotenv import load_dotenv
|
||||
|
||||
# Load .env file from project root
|
||||
PROJECT_ROOT = Path(__file__).parent.parent
|
||||
load_dotenv(PROJECT_ROOT / ".env")
|
||||
|
||||
|
||||
class Settings(BaseSettings):
|
||||
"""Application settings loaded from environment variables."""
|
||||
|
||||
# Supabase
|
||||
supabase_url: str = Field(..., alias="SUPABASE_URL")
|
||||
supabase_key: str = Field(..., alias="SUPABASE_KEY")
|
||||
|
||||
# Apify
|
||||
apify_token: str = Field(..., alias="APIFY_TOKEN")
|
||||
|
||||
# Firecrawl
|
||||
firecrawl_api_key: str = Field(..., alias="FIRECRAWL_API_KEY")
|
||||
|
||||
# Anthropic
|
||||
anthropic_api_key: str = Field(..., alias="ANTHROPIC_API_KEY")
|
||||
|
||||
# Application settings
|
||||
log_level: str = Field(default="INFO", alias="LOG_LEVEL")
|
||||
batch_size: int = Field(default=10, alias="BATCH_SIZE")
|
||||
max_retries: int = Field(default=3, alias="MAX_RETRIES")
|
||||
|
||||
# Paths
|
||||
export_dir: Path = Field(default=PROJECT_ROOT / "export")
|
||||
images_dir: Path = Field(default=PROJECT_ROOT / "static" / "images" / "companies")
|
||||
templates_dir: Path = Field(default=Path(__file__).parent / "templates")
|
||||
|
||||
class Config:
|
||||
env_file = ".env"
|
||||
env_file_encoding = "utf-8"
|
||||
extra = "ignore"
|
||||
|
||||
|
||||
def get_settings() -> Settings:
|
||||
"""Get application settings singleton."""
|
||||
return Settings()
|
||||
|
||||
|
||||
# Apify actor IDs
|
||||
APIFY_GOOGLE_MAPS_ACTOR = "nwua9Gu5YrADL7ZDj" # Google Maps Scraper
|
||||
|
||||
# Search query templates for different languages
|
||||
SEARCH_QUERIES = {
|
||||
"de": [
|
||||
"Odoo Partner {city}",
|
||||
"Odoo Implementierung {city}",
|
||||
"Odoo Beratung {city}",
|
||||
"Odoo ERP {city}",
|
||||
"Odoo Entwicklung {city}",
|
||||
],
|
||||
"en": [
|
||||
"Odoo Partner {city}",
|
||||
"Odoo Implementation {city}",
|
||||
"Odoo Consulting {city}",
|
||||
"Odoo ERP {city}",
|
||||
"Odoo Development {city}",
|
||||
],
|
||||
"es": [
|
||||
"Odoo Partner {city}",
|
||||
"Odoo Implementación {city}",
|
||||
"Odoo Consultoría {city}",
|
||||
"Odoo ERP {city}",
|
||||
"Odoo Desarrollo {city}",
|
||||
],
|
||||
"fr": [
|
||||
"Odoo Partenaire {city}",
|
||||
"Odoo Implémentation {city}",
|
||||
"Odoo Conseil {city}",
|
||||
"Odoo ERP {city}",
|
||||
"Odoo Développement {city}",
|
||||
],
|
||||
"pt": [
|
||||
"Odoo Parceiro {city}",
|
||||
"Odoo Implementação {city}",
|
||||
"Odoo Consultoria {city}",
|
||||
"Odoo ERP {city}",
|
||||
"Odoo Desenvolvimento {city}",
|
||||
],
|
||||
"ar": [
|
||||
"Odoo Partner {city}",
|
||||
"Odoo شريك {city}",
|
||||
"Odoo ERP {city}",
|
||||
],
|
||||
}
|
||||
|
||||
# Default tags for seeding
|
||||
DEFAULT_TAGS = {
|
||||
"services": [
|
||||
"Implementation",
|
||||
"Customization",
|
||||
"Training",
|
||||
"Support",
|
||||
"Migration",
|
||||
"Integration",
|
||||
"Consulting",
|
||||
"Hosting",
|
||||
"Development",
|
||||
],
|
||||
"industries": [
|
||||
"Manufacturing",
|
||||
"Retail",
|
||||
"E-commerce",
|
||||
"Healthcare",
|
||||
"Education",
|
||||
"Finance",
|
||||
"Logistics",
|
||||
"Construction",
|
||||
"Food & Beverage",
|
||||
"Professional Services",
|
||||
],
|
||||
"modules": [
|
||||
"Sales",
|
||||
"CRM",
|
||||
"Inventory",
|
||||
"Accounting",
|
||||
"Manufacturing",
|
||||
"Website",
|
||||
"E-commerce",
|
||||
"HR",
|
||||
"Project",
|
||||
"Purchase",
|
||||
"Point of Sale",
|
||||
],
|
||||
"partner_levels": [
|
||||
"Ready Partner",
|
||||
"Silver Partner",
|
||||
"Gold Partner",
|
||||
"Platinum Partner",
|
||||
],
|
||||
}
|
||||
|
||||
# Germany cities for initial rollout
|
||||
GERMANY_CITIES = [
|
||||
{"name": "Berlin", "slug": "berlin", "state": "Berlin"},
|
||||
{"name": "München", "slug": "muenchen", "state": "Bayern"},
|
||||
{"name": "Hamburg", "slug": "hamburg", "state": "Hamburg"},
|
||||
{"name": "Köln", "slug": "koeln", "state": "Nordrhein-Westfalen"},
|
||||
{"name": "Frankfurt am Main", "slug": "frankfurt", "state": "Hessen"},
|
||||
{"name": "Stuttgart", "slug": "stuttgart", "state": "Baden-Württemberg"},
|
||||
{"name": "Düsseldorf", "slug": "duesseldorf", "state": "Nordrhein-Westfalen"},
|
||||
{"name": "Leipzig", "slug": "leipzig", "state": "Sachsen"},
|
||||
{"name": "Dortmund", "slug": "dortmund", "state": "Nordrhein-Westfalen"},
|
||||
{"name": "Essen", "slug": "essen", "state": "Nordrhein-Westfalen"},
|
||||
]
|
||||
@@ -0,0 +1,31 @@
|
||||
"""Data models for Odoo Directory."""
|
||||
|
||||
from .schemas import (
|
||||
Region,
|
||||
Country,
|
||||
City,
|
||||
Tag,
|
||||
Company,
|
||||
CompanyTag,
|
||||
CityTag,
|
||||
CountryTag,
|
||||
ExecutionPlan,
|
||||
PlanStep,
|
||||
StepStatus,
|
||||
TagCategory,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"Region",
|
||||
"Country",
|
||||
"City",
|
||||
"Tag",
|
||||
"Company",
|
||||
"CompanyTag",
|
||||
"CityTag",
|
||||
"CountryTag",
|
||||
"ExecutionPlan",
|
||||
"PlanStep",
|
||||
"StepStatus",
|
||||
"TagCategory",
|
||||
]
|
||||
@@ -0,0 +1,244 @@
|
||||
"""
|
||||
Pydantic models for Odoo Directory data structures.
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
from enum import Enum
|
||||
from typing import Optional, List, Dict, Any
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import BaseModel, Field, HttpUrl
|
||||
|
||||
|
||||
class TagCategory(str, Enum):
|
||||
"""Tag category types."""
|
||||
SERVICE = "service"
|
||||
INDUSTRY = "industry"
|
||||
MODULE = "module"
|
||||
PARTNER_LEVEL = "partner_level"
|
||||
|
||||
|
||||
class StepStatus(str, Enum):
|
||||
"""Execution step status."""
|
||||
PENDING = "pending"
|
||||
RUNNING = "running"
|
||||
COMPLETED = "completed"
|
||||
FAILED = "failed"
|
||||
SKIPPED = "skipped"
|
||||
|
||||
|
||||
class Region(BaseModel):
|
||||
"""Geographic region model."""
|
||||
id: Optional[UUID] = None
|
||||
name: str
|
||||
slug: str
|
||||
created_at: Optional[datetime] = None
|
||||
|
||||
class Config:
|
||||
from_attributes = True
|
||||
|
||||
|
||||
class Country(BaseModel):
|
||||
"""Country model."""
|
||||
id: Optional[UUID] = None
|
||||
region_id: Optional[UUID] = None
|
||||
name: str
|
||||
name_local: Optional[str] = None
|
||||
code: str # ISO 3166-1 alpha-2
|
||||
slug: str
|
||||
language: str # Primary language code
|
||||
created_at: Optional[datetime] = None
|
||||
|
||||
class Config:
|
||||
from_attributes = True
|
||||
|
||||
|
||||
class City(BaseModel):
|
||||
"""City model."""
|
||||
id: Optional[UUID] = None
|
||||
country_id: Optional[UUID] = None
|
||||
name: str
|
||||
name_local: Optional[str] = None
|
||||
slug: str
|
||||
state: Optional[str] = None
|
||||
population: Optional[int] = None
|
||||
latitude: Optional[float] = None
|
||||
longitude: Optional[float] = None
|
||||
created_at: Optional[datetime] = None
|
||||
|
||||
class Config:
|
||||
from_attributes = True
|
||||
|
||||
|
||||
class Tag(BaseModel):
|
||||
"""Tag model for services, industries, modules, partner levels."""
|
||||
id: Optional[UUID] = None
|
||||
category: TagCategory
|
||||
name: str
|
||||
slug: str
|
||||
description: Optional[str] = None
|
||||
created_at: Optional[datetime] = None
|
||||
|
||||
class Config:
|
||||
from_attributes = True
|
||||
|
||||
|
||||
class Company(BaseModel):
|
||||
"""Odoo partner company model."""
|
||||
id: Optional[UUID] = None
|
||||
city_id: Optional[UUID] = None
|
||||
|
||||
# Basic info (from Google Maps)
|
||||
name: str
|
||||
slug: str
|
||||
google_place_id: Optional[str] = None
|
||||
address: Optional[str] = None
|
||||
phone: Optional[str] = None
|
||||
website: Optional[str] = None
|
||||
email: Optional[str] = None
|
||||
|
||||
# Google Maps data
|
||||
rating: Optional[float] = None
|
||||
review_count: Optional[int] = None
|
||||
latitude: Optional[float] = None
|
||||
longitude: Optional[float] = None
|
||||
|
||||
# Enriched data (from Firecrawl)
|
||||
website_markdown: Optional[str] = None
|
||||
logo_url: Optional[str] = None
|
||||
|
||||
# Generated content (from Haiku)
|
||||
description: Optional[str] = None
|
||||
content_html: Optional[str] = None
|
||||
meta_title: Optional[str] = None
|
||||
meta_description: Optional[str] = None
|
||||
|
||||
# Status tracking
|
||||
status: str = "scraped" # scraped, enriched, content_generated, exported
|
||||
last_scraped_at: Optional[datetime] = None
|
||||
last_enriched_at: Optional[datetime] = None
|
||||
last_content_at: Optional[datetime] = None
|
||||
|
||||
created_at: Optional[datetime] = None
|
||||
updated_at: Optional[datetime] = None
|
||||
|
||||
class Config:
|
||||
from_attributes = True
|
||||
|
||||
|
||||
class CompanyTag(BaseModel):
|
||||
"""Junction table for company-tag relationship."""
|
||||
company_id: UUID
|
||||
tag_id: UUID
|
||||
confidence: Optional[float] = None # AI confidence score
|
||||
created_at: Optional[datetime] = None
|
||||
|
||||
class Config:
|
||||
from_attributes = True
|
||||
|
||||
|
||||
class CityTag(BaseModel):
|
||||
"""Aggregated tag counts per city."""
|
||||
city_id: UUID
|
||||
tag_id: UUID
|
||||
company_count: int = 0
|
||||
|
||||
class Config:
|
||||
from_attributes = True
|
||||
|
||||
|
||||
class CountryTag(BaseModel):
|
||||
"""Aggregated tag counts per country."""
|
||||
country_id: UUID
|
||||
tag_id: UUID
|
||||
company_count: int = 0
|
||||
city_count: int = 0
|
||||
|
||||
class Config:
|
||||
from_attributes = True
|
||||
|
||||
|
||||
class PlanStep(BaseModel):
|
||||
"""Individual step in an execution plan."""
|
||||
id: Optional[UUID] = None
|
||||
plan_id: Optional[UUID] = None
|
||||
step_name: str # scrape, enrich, content, export
|
||||
city_id: Optional[UUID] = None
|
||||
status: StepStatus = StepStatus.PENDING
|
||||
started_at: Optional[datetime] = None
|
||||
completed_at: Optional[datetime] = None
|
||||
error_message: Optional[str] = None
|
||||
items_processed: int = 0
|
||||
items_total: int = 0
|
||||
|
||||
class Config:
|
||||
from_attributes = True
|
||||
|
||||
|
||||
class ExecutionPlan(BaseModel):
|
||||
"""Execution plan for processing a country/region."""
|
||||
id: Optional[UUID] = None
|
||||
name: str
|
||||
country_code: str
|
||||
cities: List[str] # List of city slugs
|
||||
status: StepStatus = StepStatus.PENDING
|
||||
created_at: Optional[datetime] = None
|
||||
started_at: Optional[datetime] = None
|
||||
completed_at: Optional[datetime] = None
|
||||
|
||||
# Related steps
|
||||
steps: List[PlanStep] = []
|
||||
|
||||
class Config:
|
||||
from_attributes = True
|
||||
|
||||
|
||||
# API Response models
|
||||
|
||||
class ApifyGoogleMapsResult(BaseModel):
|
||||
"""Result from Apify Google Maps scraper."""
|
||||
title: str
|
||||
address: Optional[str] = None
|
||||
phone: Optional[str] = None
|
||||
website: Optional[str] = None
|
||||
rating: Optional[float] = None
|
||||
reviewsCount: Optional[int] = None
|
||||
placeId: Optional[str] = None
|
||||
url: Optional[str] = None
|
||||
location: Optional[Dict[str, float]] = None
|
||||
|
||||
class Config:
|
||||
extra = "ignore"
|
||||
|
||||
|
||||
class FirecrawlResult(BaseModel):
|
||||
"""Result from Firecrawl website scraper."""
|
||||
markdown: Optional[str] = None
|
||||
metadata: Optional[Dict[str, Any]] = None
|
||||
images: Optional[List[str]] = None
|
||||
|
||||
class Config:
|
||||
extra = "ignore"
|
||||
|
||||
|
||||
class HaikuTagsResponse(BaseModel):
|
||||
"""Response from Haiku tag extraction."""
|
||||
services: List[str] = []
|
||||
industries: List[str] = []
|
||||
modules: List[str] = []
|
||||
partner_level: Optional[str] = None
|
||||
confidence: float = 0.0
|
||||
|
||||
class Config:
|
||||
extra = "ignore"
|
||||
|
||||
|
||||
class HaikuContentResponse(BaseModel):
|
||||
"""Response from Haiku content generation."""
|
||||
description: str
|
||||
content_html: str
|
||||
meta_title: str
|
||||
meta_description: str
|
||||
|
||||
class Config:
|
||||
extra = "ignore"
|
||||
@@ -0,0 +1,17 @@
|
||||
"""Service modules for external API integrations."""
|
||||
|
||||
from .supabase import SupabaseService
|
||||
from .apify import ApifyService
|
||||
from .firecrawl import FirecrawlService
|
||||
from .haiku import HaikuService
|
||||
from .images import ImageService
|
||||
from .tags import TagAggregationService
|
||||
|
||||
__all__ = [
|
||||
"SupabaseService",
|
||||
"ApifyService",
|
||||
"FirecrawlService",
|
||||
"HaikuService",
|
||||
"ImageService",
|
||||
"TagAggregationService",
|
||||
]
|
||||
@@ -0,0 +1,224 @@
|
||||
"""
|
||||
Apify service for Google Maps scraping.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from typing import List, Optional, Dict, Any
|
||||
import time
|
||||
|
||||
import httpx
|
||||
|
||||
from ..config import get_settings, APIFY_GOOGLE_MAPS_ACTOR, SEARCH_QUERIES
|
||||
from ..models.schemas import ApifyGoogleMapsResult
|
||||
from ..utils.retry import retry_with_backoff
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ApifyService:
|
||||
"""Service for interacting with Apify Google Maps scraper."""
|
||||
|
||||
BASE_URL = "https://api.apify.com/v2"
|
||||
|
||||
def __init__(self):
|
||||
"""Initialize Apify client."""
|
||||
settings = get_settings()
|
||||
self.token = settings.apify_token
|
||||
self.headers = {"Authorization": f"Bearer {self.token}"}
|
||||
|
||||
def _get_search_queries(self, city: str, language: str = "de") -> List[str]:
|
||||
"""
|
||||
Generate search queries for a city.
|
||||
|
||||
Args:
|
||||
city: City name
|
||||
language: Language code for queries
|
||||
|
||||
Returns:
|
||||
List of search query strings
|
||||
"""
|
||||
templates = SEARCH_QUERIES.get(language, SEARCH_QUERIES["en"])
|
||||
return [template.format(city=city) for template in templates]
|
||||
|
||||
@retry_with_backoff(max_attempts=3, exceptions=(httpx.HTTPError,))
|
||||
def run_google_maps_scraper(
|
||||
self,
|
||||
search_queries: List[str],
|
||||
max_results_per_query: int = 20,
|
||||
language: str = "de",
|
||||
country: str = "DE",
|
||||
) -> str:
|
||||
"""
|
||||
Start a Google Maps scraper run.
|
||||
|
||||
Args:
|
||||
search_queries: List of search queries
|
||||
max_results_per_query: Max results per query
|
||||
language: Language for results
|
||||
country: Country code for search
|
||||
|
||||
Returns:
|
||||
Run ID for tracking
|
||||
"""
|
||||
url = f"{self.BASE_URL}/acts/{APIFY_GOOGLE_MAPS_ACTOR}/runs"
|
||||
|
||||
payload = {
|
||||
"searchStringsArray": search_queries,
|
||||
"maxCrawledPlaces": max_results_per_query * len(search_queries),
|
||||
"language": language,
|
||||
"countryCode": country,
|
||||
"maxReviews": 0, # Don't fetch reviews to save credits
|
||||
"maxImages": 0, # Don't fetch images
|
||||
"includeWebResults": False,
|
||||
}
|
||||
|
||||
with httpx.Client(timeout=60.0) as client:
|
||||
response = client.post(url, json=payload, headers=self.headers)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
run_id = data["data"]["id"]
|
||||
logger.info(f"Started Apify run: {run_id}")
|
||||
return run_id
|
||||
|
||||
def wait_for_run(
|
||||
self, run_id: str, timeout: int = 300, poll_interval: int = 10
|
||||
) -> str:
|
||||
"""
|
||||
Wait for an Apify run to complete.
|
||||
|
||||
Args:
|
||||
run_id: Run ID to wait for
|
||||
timeout: Maximum wait time in seconds
|
||||
poll_interval: Time between status checks
|
||||
|
||||
Returns:
|
||||
Final status of the run
|
||||
"""
|
||||
url = f"{self.BASE_URL}/actor-runs/{run_id}"
|
||||
start_time = time.time()
|
||||
|
||||
while True:
|
||||
if time.time() - start_time > timeout:
|
||||
raise TimeoutError(f"Apify run {run_id} timed out after {timeout}s")
|
||||
|
||||
with httpx.Client(timeout=30.0) as client:
|
||||
response = client.get(url, headers=self.headers)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
|
||||
status = data["data"]["status"]
|
||||
logger.debug(f"Run {run_id} status: {status}")
|
||||
|
||||
if status in ("SUCCEEDED", "FAILED", "ABORTED", "TIMED-OUT"):
|
||||
return status
|
||||
|
||||
time.sleep(poll_interval)
|
||||
|
||||
def get_run_results(self, run_id: str) -> List[ApifyGoogleMapsResult]:
|
||||
"""
|
||||
Get results from a completed Apify run.
|
||||
|
||||
Args:
|
||||
run_id: Run ID to get results for
|
||||
|
||||
Returns:
|
||||
List of Google Maps results
|
||||
"""
|
||||
url = f"{self.BASE_URL}/actor-runs/{run_id}/dataset/items"
|
||||
|
||||
with httpx.Client(timeout=60.0) as client:
|
||||
response = client.get(url, headers=self.headers)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
|
||||
results = []
|
||||
for item in data:
|
||||
try:
|
||||
result = ApifyGoogleMapsResult(
|
||||
title=item.get("title", ""),
|
||||
address=item.get("address"),
|
||||
phone=item.get("phone"),
|
||||
website=item.get("website"),
|
||||
rating=item.get("totalScore"),
|
||||
reviewsCount=item.get("reviewsCount"),
|
||||
placeId=item.get("placeId"),
|
||||
url=item.get("url"),
|
||||
location=item.get("location"),
|
||||
)
|
||||
results.append(result)
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to parse result: {e}")
|
||||
|
||||
logger.info(f"Retrieved {len(results)} results from run {run_id}")
|
||||
return results
|
||||
|
||||
def scrape_city(
|
||||
self,
|
||||
city_name: str,
|
||||
language: str = "de",
|
||||
country: str = "DE",
|
||||
max_per_query: int = 20,
|
||||
) -> List[ApifyGoogleMapsResult]:
|
||||
"""
|
||||
Scrape Google Maps for Odoo partners in a city.
|
||||
|
||||
This is the main high-level method that combines query generation,
|
||||
running the scraper, and retrieving results.
|
||||
|
||||
Args:
|
||||
city_name: Name of the city to scrape
|
||||
language: Language code
|
||||
country: Country code
|
||||
max_per_query: Max results per search query
|
||||
|
||||
Returns:
|
||||
List of Google Maps results
|
||||
"""
|
||||
queries = self._get_search_queries(city_name, language)
|
||||
logger.info(f"Scraping {city_name} with {len(queries)} queries")
|
||||
|
||||
# Start the run
|
||||
run_id = self.run_google_maps_scraper(
|
||||
search_queries=queries,
|
||||
max_results_per_query=max_per_query,
|
||||
language=language,
|
||||
country=country,
|
||||
)
|
||||
|
||||
# Wait for completion
|
||||
status = self.wait_for_run(run_id)
|
||||
|
||||
if status != "SUCCEEDED":
|
||||
raise RuntimeError(f"Apify run failed with status: {status}")
|
||||
|
||||
# Get results
|
||||
return self.get_run_results(run_id)
|
||||
|
||||
def deduplicate_results(
|
||||
self, results: List[ApifyGoogleMapsResult]
|
||||
) -> List[ApifyGoogleMapsResult]:
|
||||
"""
|
||||
Remove duplicate results based on place ID.
|
||||
|
||||
Args:
|
||||
results: List of results to deduplicate
|
||||
|
||||
Returns:
|
||||
Deduplicated list
|
||||
"""
|
||||
seen_place_ids = set()
|
||||
unique_results = []
|
||||
|
||||
for result in results:
|
||||
if result.placeId and result.placeId not in seen_place_ids:
|
||||
seen_place_ids.add(result.placeId)
|
||||
unique_results.append(result)
|
||||
elif not result.placeId:
|
||||
# Keep results without place ID but log warning
|
||||
logger.warning(f"Result without place ID: {result.title}")
|
||||
unique_results.append(result)
|
||||
|
||||
logger.info(
|
||||
f"Deduplicated {len(results)} results to {len(unique_results)} unique"
|
||||
)
|
||||
return unique_results
|
||||
@@ -0,0 +1,177 @@
|
||||
"""
|
||||
Firecrawl service for website scraping and enrichment.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from typing import Optional, Dict, Any, List
|
||||
|
||||
import httpx
|
||||
|
||||
from ..config import get_settings
|
||||
from ..models.schemas import FirecrawlResult
|
||||
from ..utils.retry import retry_with_backoff
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class FirecrawlService:
|
||||
"""Service for scraping websites using Firecrawl."""
|
||||
|
||||
BASE_URL = "https://api.firecrawl.dev/v1"
|
||||
|
||||
def __init__(self):
|
||||
"""Initialize Firecrawl client."""
|
||||
settings = get_settings()
|
||||
self.api_key = settings.firecrawl_api_key
|
||||
self.headers = {
|
||||
"Authorization": f"Bearer {self.api_key}",
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
|
||||
@retry_with_backoff(max_attempts=3, exceptions=(httpx.HTTPError,))
|
||||
def scrape_url(
|
||||
self,
|
||||
url: str,
|
||||
formats: Optional[List[str]] = None,
|
||||
include_images: bool = True,
|
||||
) -> FirecrawlResult:
|
||||
"""
|
||||
Scrape a single URL.
|
||||
|
||||
Args:
|
||||
url: URL to scrape
|
||||
formats: Output formats (default: markdown)
|
||||
include_images: Whether to extract image URLs
|
||||
|
||||
Returns:
|
||||
Scraped content
|
||||
"""
|
||||
if formats is None:
|
||||
formats = ["markdown"]
|
||||
|
||||
endpoint = f"{self.BASE_URL}/scrape"
|
||||
|
||||
payload = {
|
||||
"url": url,
|
||||
"formats": formats,
|
||||
}
|
||||
|
||||
with httpx.Client(timeout=60.0) as client:
|
||||
response = client.post(endpoint, json=payload, headers=self.headers)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
|
||||
if not data.get("success"):
|
||||
raise RuntimeError(f"Firecrawl scrape failed: {data.get('error')}")
|
||||
|
||||
result_data = data.get("data", {})
|
||||
|
||||
# Extract images from metadata if available
|
||||
images = []
|
||||
if include_images:
|
||||
metadata = result_data.get("metadata", {})
|
||||
# Try to get og:image and other image sources
|
||||
if metadata.get("ogImage"):
|
||||
images.append(metadata["ogImage"])
|
||||
if metadata.get("logo"):
|
||||
images.append(metadata["logo"])
|
||||
|
||||
return FirecrawlResult(
|
||||
markdown=result_data.get("markdown"),
|
||||
metadata=result_data.get("metadata"),
|
||||
images=images,
|
||||
)
|
||||
|
||||
def extract_company_info(
|
||||
self, url: str
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Extract structured company information from a website.
|
||||
|
||||
Args:
|
||||
url: Company website URL
|
||||
|
||||
Returns:
|
||||
Dictionary with extracted information
|
||||
"""
|
||||
try:
|
||||
result = self.scrape_url(url, formats=["markdown"], include_images=True)
|
||||
|
||||
info = {
|
||||
"markdown": result.markdown,
|
||||
"images": result.images or [],
|
||||
"logo_url": None,
|
||||
"email": None,
|
||||
"title": None,
|
||||
"description": None,
|
||||
}
|
||||
|
||||
# Extract metadata
|
||||
if result.metadata:
|
||||
info["title"] = result.metadata.get("title")
|
||||
info["description"] = result.metadata.get("description")
|
||||
if result.metadata.get("ogImage"):
|
||||
info["logo_url"] = result.metadata["ogImage"]
|
||||
|
||||
# Try to find email in markdown content
|
||||
if result.markdown:
|
||||
import re
|
||||
email_pattern = r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}"
|
||||
emails = re.findall(email_pattern, result.markdown)
|
||||
if emails:
|
||||
# Filter out common non-company emails
|
||||
company_emails = [
|
||||
e for e in emails
|
||||
if not any(
|
||||
domain in e.lower()
|
||||
for domain in ["example.com", "test.com", "email.com"]
|
||||
)
|
||||
]
|
||||
if company_emails:
|
||||
info["email"] = company_emails[0]
|
||||
|
||||
return info
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to scrape {url}: {e}")
|
||||
return {
|
||||
"markdown": None,
|
||||
"images": [],
|
||||
"logo_url": None,
|
||||
"email": None,
|
||||
"title": None,
|
||||
"description": None,
|
||||
"error": str(e),
|
||||
}
|
||||
|
||||
def batch_scrape(
|
||||
self, urls: List[str], batch_size: int = 5
|
||||
) -> Dict[str, FirecrawlResult]:
|
||||
"""
|
||||
Scrape multiple URLs in batches.
|
||||
|
||||
Args:
|
||||
urls: List of URLs to scrape
|
||||
batch_size: Number of concurrent scrapes
|
||||
|
||||
Returns:
|
||||
Dictionary mapping URLs to results
|
||||
"""
|
||||
results = {}
|
||||
|
||||
for i in range(0, len(urls), batch_size):
|
||||
batch = urls[i : i + batch_size]
|
||||
logger.info(f"Scraping batch {i // batch_size + 1} ({len(batch)} URLs)")
|
||||
|
||||
for url in batch:
|
||||
try:
|
||||
results[url] = self.scrape_url(url)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to scrape {url}: {e}")
|
||||
results[url] = FirecrawlResult(
|
||||
markdown=None,
|
||||
metadata={"error": str(e)},
|
||||
images=[],
|
||||
)
|
||||
|
||||
return results
|
||||
@@ -0,0 +1,359 @@
|
||||
"""
|
||||
Claude Haiku service for content generation.
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
from typing import Optional, List, Dict, Any
|
||||
|
||||
from anthropic import Anthropic
|
||||
|
||||
from ..config import get_settings
|
||||
from ..models.schemas import HaikuTagsResponse, HaikuContentResponse, TagCategory
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class HaikuService:
|
||||
"""Service for AI content generation using Claude Haiku."""
|
||||
|
||||
def __init__(self):
|
||||
"""Initialize Anthropic client."""
|
||||
settings = get_settings()
|
||||
self.client = Anthropic(api_key=settings.anthropic_api_key)
|
||||
self.model = "claude-3-5-haiku-20241022"
|
||||
|
||||
def _call_haiku(
|
||||
self,
|
||||
system_prompt: str,
|
||||
user_prompt: str,
|
||||
max_tokens: int = 4096,
|
||||
) -> str:
|
||||
"""
|
||||
Make a call to Claude Haiku.
|
||||
|
||||
Args:
|
||||
system_prompt: System instructions
|
||||
user_prompt: User message
|
||||
max_tokens: Maximum response tokens
|
||||
|
||||
Returns:
|
||||
Response text
|
||||
"""
|
||||
response = self.client.messages.create(
|
||||
model=self.model,
|
||||
max_tokens=max_tokens,
|
||||
system=system_prompt,
|
||||
messages=[{"role": "user", "content": user_prompt}],
|
||||
)
|
||||
return response.content[0].text
|
||||
|
||||
def extract_tags(
|
||||
self,
|
||||
company_name: str,
|
||||
website_markdown: Optional[str],
|
||||
address: Optional[str] = None,
|
||||
) -> HaikuTagsResponse:
|
||||
"""
|
||||
Extract tags from company information.
|
||||
|
||||
Args:
|
||||
company_name: Company name
|
||||
website_markdown: Scraped website content
|
||||
address: Company address
|
||||
|
||||
Returns:
|
||||
Extracted tags with confidence
|
||||
"""
|
||||
system_prompt = """You are an expert at analyzing Odoo partner companies.
|
||||
Extract relevant tags from the provided company information.
|
||||
|
||||
Respond with valid JSON only, no other text. Use this exact structure:
|
||||
{
|
||||
"services": ["list of services they offer"],
|
||||
"industries": ["list of industries they serve"],
|
||||
"modules": ["list of Odoo modules they specialize in"],
|
||||
"partner_level": "Ready Partner|Silver Partner|Gold Partner|Platinum Partner or null",
|
||||
"confidence": 0.0 to 1.0
|
||||
}
|
||||
|
||||
Valid services: Implementation, Customization, Training, Support, Migration, Integration, Consulting, Hosting, Development
|
||||
Valid industries: Manufacturing, Retail, E-commerce, Healthcare, Education, Finance, Logistics, Construction, Food & Beverage, Professional Services
|
||||
Valid modules: Sales, CRM, Inventory, Accounting, Manufacturing, Website, E-commerce, HR, Project, Purchase, Point of Sale
|
||||
|
||||
Only include tags you can confidently identify from the content."""
|
||||
|
||||
user_prompt = f"""Company: {company_name}
|
||||
Address: {address or "Not provided"}
|
||||
|
||||
Website content:
|
||||
{website_markdown[:8000] if website_markdown else "No website content available"}
|
||||
|
||||
Extract tags for this Odoo partner company."""
|
||||
|
||||
try:
|
||||
response = self._call_haiku(system_prompt, user_prompt, max_tokens=1024)
|
||||
# Parse JSON response
|
||||
data = json.loads(response)
|
||||
return HaikuTagsResponse(**data)
|
||||
except json.JSONDecodeError as e:
|
||||
logger.error(f"Failed to parse tags response: {e}")
|
||||
return HaikuTagsResponse(confidence=0.0)
|
||||
except Exception as e:
|
||||
logger.error(f"Tag extraction failed: {e}")
|
||||
return HaikuTagsResponse(confidence=0.0)
|
||||
|
||||
def generate_company_content(
|
||||
self,
|
||||
company_name: str,
|
||||
city_name: str,
|
||||
country_name: str,
|
||||
website_markdown: Optional[str],
|
||||
services: List[str],
|
||||
industries: List[str],
|
||||
modules: List[str],
|
||||
partner_level: Optional[str] = None,
|
||||
language: str = "de",
|
||||
) -> HaikuContentResponse:
|
||||
"""
|
||||
Generate SEO-optimized company page content.
|
||||
|
||||
Args:
|
||||
company_name: Company name
|
||||
city_name: City name
|
||||
country_name: Country name
|
||||
website_markdown: Scraped website content
|
||||
services: List of service tags
|
||||
industries: List of industry tags
|
||||
modules: List of module tags
|
||||
partner_level: Partner certification level
|
||||
language: Output language code
|
||||
|
||||
Returns:
|
||||
Generated content
|
||||
"""
|
||||
lang_instructions = {
|
||||
"de": "Write all content in German (Deutsch).",
|
||||
"en": "Write all content in English.",
|
||||
"es": "Write all content in Spanish (Español).",
|
||||
"fr": "Write all content in French (Français).",
|
||||
"pt": "Write all content in Portuguese (Português).",
|
||||
"ar": "Write all content in Arabic (العربية).",
|
||||
}
|
||||
|
||||
system_prompt = f"""You are an expert SEO content writer for Odoo partner directories.
|
||||
{lang_instructions.get(language, lang_instructions["en"])}
|
||||
|
||||
Create engaging, informative content about an Odoo partner company.
|
||||
The content should be at least 800 words and SEO-optimized.
|
||||
|
||||
Respond with valid JSON only:
|
||||
{{
|
||||
"description": "2-3 sentence company description",
|
||||
"content_html": "Full HTML content with headings, paragraphs, lists",
|
||||
"meta_title": "SEO title (50-60 chars)",
|
||||
"meta_description": "SEO meta description (150-160 chars)"
|
||||
}}
|
||||
|
||||
Include:
|
||||
- Company overview
|
||||
- Services offered
|
||||
- Industries served
|
||||
- Odoo modules expertise
|
||||
- Why choose this partner
|
||||
- Location information
|
||||
|
||||
Use semantic HTML: <h2>, <h3>, <p>, <ul>, <li>, <strong>"""
|
||||
|
||||
tags_info = f"""
|
||||
Services: {", ".join(services) if services else "Various Odoo services"}
|
||||
Industries: {", ".join(industries) if industries else "Multiple industries"}
|
||||
Modules: {", ".join(modules) if modules else "Various Odoo modules"}
|
||||
Partner Level: {partner_level or "Odoo Partner"}"""
|
||||
|
||||
user_prompt = f"""Create content for:
|
||||
|
||||
Company: {company_name}
|
||||
Location: {city_name}, {country_name}
|
||||
{tags_info}
|
||||
|
||||
Website information:
|
||||
{website_markdown[:6000] if website_markdown else "Limited information available"}
|
||||
|
||||
Generate comprehensive SEO content for this Odoo partner's directory page."""
|
||||
|
||||
try:
|
||||
response = self._call_haiku(system_prompt, user_prompt, max_tokens=4096)
|
||||
data = json.loads(response)
|
||||
return HaikuContentResponse(**data)
|
||||
except json.JSONDecodeError as e:
|
||||
logger.error(f"Failed to parse content response: {e}")
|
||||
# Return fallback content
|
||||
return self._generate_fallback_content(
|
||||
company_name, city_name, country_name, services, language
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Content generation failed: {e}")
|
||||
return self._generate_fallback_content(
|
||||
company_name, city_name, country_name, services, language
|
||||
)
|
||||
|
||||
def _generate_fallback_content(
|
||||
self,
|
||||
company_name: str,
|
||||
city_name: str,
|
||||
country_name: str,
|
||||
services: List[str],
|
||||
language: str,
|
||||
) -> HaikuContentResponse:
|
||||
"""Generate basic fallback content when AI generation fails."""
|
||||
services_text = ", ".join(services) if services else "Odoo implementation"
|
||||
|
||||
if language == "de":
|
||||
description = f"{company_name} ist ein Odoo Partner in {city_name}, {country_name}."
|
||||
content = f"""<h2>Über {company_name}</h2>
|
||||
<p>{company_name} ist ein zertifizierter Odoo Partner mit Sitz in {city_name}.
|
||||
Das Unternehmen bietet professionelle Odoo-Dienstleistungen für Unternehmen in der Region.</p>
|
||||
|
||||
<h3>Dienstleistungen</h3>
|
||||
<p>Als Odoo Partner bietet {company_name} folgende Leistungen: {services_text}.</p>
|
||||
|
||||
<h3>Standort</h3>
|
||||
<p>{company_name} befindet sich in {city_name}, {country_name} und betreut Kunden in der gesamten Region.</p>"""
|
||||
meta_title = f"{company_name} - Odoo Partner in {city_name}"
|
||||
meta_desc = f"{company_name} ist Ihr Odoo Partner in {city_name}. Professionelle ERP-Beratung und Implementierung."
|
||||
else:
|
||||
description = f"{company_name} is an Odoo partner located in {city_name}, {country_name}."
|
||||
content = f"""<h2>About {company_name}</h2>
|
||||
<p>{company_name} is a certified Odoo partner based in {city_name}.
|
||||
The company provides professional Odoo services for businesses in the region.</p>
|
||||
|
||||
<h3>Services</h3>
|
||||
<p>As an Odoo partner, {company_name} offers: {services_text}.</p>
|
||||
|
||||
<h3>Location</h3>
|
||||
<p>{company_name} is located in {city_name}, {country_name} and serves clients throughout the region.</p>"""
|
||||
meta_title = f"{company_name} - Odoo Partner in {city_name}"
|
||||
meta_desc = f"{company_name} is your Odoo partner in {city_name}. Professional ERP consulting and implementation."
|
||||
|
||||
return HaikuContentResponse(
|
||||
description=description,
|
||||
content_html=content,
|
||||
meta_title=meta_title[:60],
|
||||
meta_description=meta_desc[:160],
|
||||
)
|
||||
|
||||
def generate_city_hub_content(
|
||||
self,
|
||||
city_name: str,
|
||||
country_name: str,
|
||||
company_count: int,
|
||||
top_services: List[str],
|
||||
language: str = "de",
|
||||
) -> Dict[str, str]:
|
||||
"""
|
||||
Generate city hub page content.
|
||||
|
||||
Args:
|
||||
city_name: City name
|
||||
country_name: Country name
|
||||
company_count: Number of companies in city
|
||||
top_services: Most common services
|
||||
language: Output language
|
||||
|
||||
Returns:
|
||||
Dictionary with content sections
|
||||
"""
|
||||
lang_instructions = {
|
||||
"de": "Write in German.",
|
||||
"en": "Write in English.",
|
||||
}
|
||||
|
||||
system_prompt = f"""You are an SEO expert writing city hub pages for an Odoo partner directory.
|
||||
{lang_instructions.get(language, "Write in English.")}
|
||||
|
||||
Create compelling content (1000+ words) about finding Odoo partners in this city.
|
||||
|
||||
Return valid JSON:
|
||||
{{
|
||||
"intro": "Introduction paragraph about Odoo partners in this city",
|
||||
"content_html": "Full HTML content about Odoo services in this city",
|
||||
"meta_title": "SEO title (50-60 chars)",
|
||||
"meta_description": "Meta description (150-160 chars)"
|
||||
}}"""
|
||||
|
||||
user_prompt = f"""City: {city_name}, {country_name}
|
||||
Number of Odoo Partners: {company_count}
|
||||
Top Services: {", ".join(top_services)}
|
||||
|
||||
Create engaging city hub content for this Odoo partner directory."""
|
||||
|
||||
try:
|
||||
response = self._call_haiku(system_prompt, user_prompt)
|
||||
return json.loads(response)
|
||||
except Exception as e:
|
||||
logger.error(f"City hub content generation failed: {e}")
|
||||
return {
|
||||
"intro": f"Find Odoo partners in {city_name}.",
|
||||
"content_html": f"<p>Discover {company_count} Odoo partners in {city_name}, {country_name}.</p>",
|
||||
"meta_title": f"Odoo Partners in {city_name}",
|
||||
"meta_description": f"Find {company_count} certified Odoo partners in {city_name}, {country_name}.",
|
||||
}
|
||||
|
||||
def generate_country_hub_content(
|
||||
self,
|
||||
country_name: str,
|
||||
city_count: int,
|
||||
company_count: int,
|
||||
top_cities: List[str],
|
||||
language: str = "de",
|
||||
) -> Dict[str, str]:
|
||||
"""
|
||||
Generate country hub page content.
|
||||
|
||||
Args:
|
||||
country_name: Country name
|
||||
city_count: Number of cities with partners
|
||||
company_count: Total companies in country
|
||||
top_cities: Cities with most partners
|
||||
language: Output language
|
||||
|
||||
Returns:
|
||||
Dictionary with content sections
|
||||
"""
|
||||
lang_instructions = {
|
||||
"de": "Write in German.",
|
||||
"en": "Write in English.",
|
||||
}
|
||||
|
||||
system_prompt = f"""You are an SEO expert writing country hub pages for an Odoo partner directory.
|
||||
{lang_instructions.get(language, "Write in English.")}
|
||||
|
||||
Create comprehensive content (1200+ words) about Odoo partners in this country.
|
||||
|
||||
Return valid JSON:
|
||||
{{
|
||||
"intro": "Introduction about Odoo partners in this country",
|
||||
"content_html": "Full HTML content about the Odoo ecosystem in this country",
|
||||
"meta_title": "SEO title (50-60 chars)",
|
||||
"meta_description": "Meta description (150-160 chars)"
|
||||
}}"""
|
||||
|
||||
user_prompt = f"""Country: {country_name}
|
||||
Number of Cities: {city_count}
|
||||
Total Odoo Partners: {company_count}
|
||||
Top Cities: {", ".join(top_cities)}
|
||||
|
||||
Create engaging country hub content for this Odoo partner directory."""
|
||||
|
||||
try:
|
||||
response = self._call_haiku(system_prompt, user_prompt)
|
||||
return json.loads(response)
|
||||
except Exception as e:
|
||||
logger.error(f"Country hub content generation failed: {e}")
|
||||
return {
|
||||
"intro": f"Find Odoo partners across {country_name}.",
|
||||
"content_html": f"<p>Discover {company_count} Odoo partners in {city_count} cities across {country_name}.</p>",
|
||||
"meta_title": f"Odoo Partners in {country_name}",
|
||||
"meta_description": f"Find {company_count} certified Odoo partners in {city_count} cities across {country_name}.",
|
||||
}
|
||||
@@ -0,0 +1,223 @@
|
||||
"""
|
||||
Image processing service for company logos and images.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import Optional, Tuple
|
||||
from urllib.parse import urlparse
|
||||
import hashlib
|
||||
|
||||
import httpx
|
||||
from PIL import Image
|
||||
import io
|
||||
|
||||
from ..config import get_settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ImageService:
|
||||
"""Service for downloading and processing images."""
|
||||
|
||||
# Standard sizes for company images
|
||||
SIZES = {
|
||||
"thumbnail": (150, 150),
|
||||
"medium": (300, 300),
|
||||
"large": (600, 600),
|
||||
}
|
||||
|
||||
def __init__(self):
|
||||
"""Initialize image service."""
|
||||
settings = get_settings()
|
||||
self.images_dir = settings.images_dir
|
||||
self.images_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
def _get_image_hash(self, url: str) -> str:
|
||||
"""Generate a short hash for an image URL."""
|
||||
return hashlib.md5(url.encode()).hexdigest()[:12]
|
||||
|
||||
def _get_image_path(
|
||||
self, company_slug: str, image_type: str = "logo", size: str = "medium"
|
||||
) -> Path:
|
||||
"""
|
||||
Get the path for storing a company image.
|
||||
|
||||
Args:
|
||||
company_slug: Company slug for filename
|
||||
image_type: Type of image (logo, photo)
|
||||
size: Size variant (thumbnail, medium, large)
|
||||
|
||||
Returns:
|
||||
Path to image file
|
||||
"""
|
||||
filename = f"{company_slug}_{image_type}_{size}.webp"
|
||||
return self.images_dir / filename
|
||||
|
||||
def download_image(self, url: str) -> Optional[bytes]:
|
||||
"""
|
||||
Download an image from URL.
|
||||
|
||||
Args:
|
||||
url: Image URL
|
||||
|
||||
Returns:
|
||||
Image bytes or None if download failed
|
||||
"""
|
||||
try:
|
||||
with httpx.Client(timeout=30.0, follow_redirects=True) as client:
|
||||
response = client.get(url)
|
||||
response.raise_for_status()
|
||||
|
||||
content_type = response.headers.get("content-type", "")
|
||||
if not content_type.startswith("image/"):
|
||||
logger.warning(f"URL is not an image: {url}")
|
||||
return None
|
||||
|
||||
return response.content
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to download image from {url}: {e}")
|
||||
return None
|
||||
|
||||
def process_image(
|
||||
self,
|
||||
image_data: bytes,
|
||||
size: Tuple[int, int],
|
||||
quality: int = 85,
|
||||
) -> Optional[bytes]:
|
||||
"""
|
||||
Process and convert an image to WebP format.
|
||||
|
||||
Args:
|
||||
image_data: Raw image bytes
|
||||
size: Target size (width, height)
|
||||
quality: WebP quality (1-100)
|
||||
|
||||
Returns:
|
||||
Processed image bytes or None if processing failed
|
||||
"""
|
||||
try:
|
||||
# Open image
|
||||
img = Image.open(io.BytesIO(image_data))
|
||||
|
||||
# Convert to RGB if necessary (for WebP)
|
||||
if img.mode in ("RGBA", "P"):
|
||||
# Create white background for transparency
|
||||
background = Image.new("RGB", img.size, (255, 255, 255))
|
||||
if img.mode == "P":
|
||||
img = img.convert("RGBA")
|
||||
background.paste(img, mask=img.split()[-1])
|
||||
img = background
|
||||
elif img.mode != "RGB":
|
||||
img = img.convert("RGB")
|
||||
|
||||
# Resize maintaining aspect ratio
|
||||
img.thumbnail(size, Image.Resampling.LANCZOS)
|
||||
|
||||
# Save to WebP
|
||||
output = io.BytesIO()
|
||||
img.save(output, format="WEBP", quality=quality, method=6)
|
||||
return output.getvalue()
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to process image: {e}")
|
||||
return None
|
||||
|
||||
def save_company_logo(
|
||||
self,
|
||||
company_slug: str,
|
||||
logo_url: str,
|
||||
sizes: Optional[list] = None,
|
||||
) -> dict:
|
||||
"""
|
||||
Download and save company logo in multiple sizes.
|
||||
|
||||
Args:
|
||||
company_slug: Company slug for filename
|
||||
logo_url: URL of the logo image
|
||||
sizes: List of size names to generate (default: all)
|
||||
|
||||
Returns:
|
||||
Dictionary mapping size names to file paths
|
||||
"""
|
||||
if sizes is None:
|
||||
sizes = list(self.SIZES.keys())
|
||||
|
||||
saved_paths = {}
|
||||
|
||||
# Download original
|
||||
image_data = self.download_image(logo_url)
|
||||
if not image_data:
|
||||
return saved_paths
|
||||
|
||||
# Process and save each size
|
||||
for size_name in sizes:
|
||||
if size_name not in self.SIZES:
|
||||
continue
|
||||
|
||||
size = self.SIZES[size_name]
|
||||
processed = self.process_image(image_data, size)
|
||||
|
||||
if processed:
|
||||
path = self._get_image_path(company_slug, "logo", size_name)
|
||||
path.write_bytes(processed)
|
||||
saved_paths[size_name] = str(path)
|
||||
logger.debug(f"Saved {size_name} logo: {path}")
|
||||
|
||||
return saved_paths
|
||||
|
||||
def get_logo_urls(self, company_slug: str, base_url: str = "") -> dict:
|
||||
"""
|
||||
Get URLs for company logo images.
|
||||
|
||||
Args:
|
||||
company_slug: Company slug
|
||||
base_url: Base URL for the images (e.g., /static/images/companies/)
|
||||
|
||||
Returns:
|
||||
Dictionary mapping size names to URLs
|
||||
"""
|
||||
urls = {}
|
||||
for size_name in self.SIZES:
|
||||
path = self._get_image_path(company_slug, "logo", size_name)
|
||||
if path.exists():
|
||||
relative_path = path.relative_to(self.images_dir)
|
||||
urls[size_name] = f"{base_url}{relative_path}"
|
||||
return urls
|
||||
|
||||
def cleanup_company_images(self, company_slug: str) -> int:
|
||||
"""
|
||||
Remove all images for a company.
|
||||
|
||||
Args:
|
||||
company_slug: Company slug
|
||||
|
||||
Returns:
|
||||
Number of files removed
|
||||
"""
|
||||
removed = 0
|
||||
for file in self.images_dir.glob(f"{company_slug}_*"):
|
||||
file.unlink()
|
||||
removed += 1
|
||||
return removed
|
||||
|
||||
def get_storage_stats(self) -> dict:
|
||||
"""
|
||||
Get storage statistics for the images directory.
|
||||
|
||||
Returns:
|
||||
Dictionary with stats
|
||||
"""
|
||||
total_size = 0
|
||||
file_count = 0
|
||||
|
||||
for file in self.images_dir.glob("*.webp"):
|
||||
total_size += file.stat().st_size
|
||||
file_count += 1
|
||||
|
||||
return {
|
||||
"file_count": file_count,
|
||||
"total_size_mb": round(total_size / (1024 * 1024), 2),
|
||||
"directory": str(self.images_dir),
|
||||
}
|
||||
@@ -0,0 +1,472 @@
|
||||
"""
|
||||
Supabase database service for Odoo Directory.
|
||||
Handles all database operations.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from typing import Optional, List, Dict, Any
|
||||
from uuid import UUID
|
||||
|
||||
from supabase import create_client, Client
|
||||
|
||||
from ..config import get_settings
|
||||
from ..models.schemas import (
|
||||
Region,
|
||||
Country,
|
||||
City,
|
||||
Tag,
|
||||
Company,
|
||||
CompanyTag,
|
||||
CityTag,
|
||||
CountryTag,
|
||||
ExecutionPlan,
|
||||
PlanStep,
|
||||
TagCategory,
|
||||
StepStatus,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class SupabaseService:
|
||||
"""Service for Supabase database operations."""
|
||||
|
||||
def __init__(self):
|
||||
"""Initialize Supabase client."""
|
||||
settings = get_settings()
|
||||
self.client: Client = create_client(
|
||||
settings.supabase_url, settings.supabase_key
|
||||
)
|
||||
|
||||
# ==================== Region Operations ====================
|
||||
|
||||
def create_region(self, region: Region) -> Region:
|
||||
"""Create a new region."""
|
||||
data = {"name": region.name, "slug": region.slug}
|
||||
result = self.client.table("regions").insert(data).execute()
|
||||
return Region(**result.data[0])
|
||||
|
||||
def get_region_by_slug(self, slug: str) -> Optional[Region]:
|
||||
"""Get region by slug."""
|
||||
result = (
|
||||
self.client.table("regions").select("*").eq("slug", slug).execute()
|
||||
)
|
||||
if result.data:
|
||||
return Region(**result.data[0])
|
||||
return None
|
||||
|
||||
def list_regions(self) -> List[Region]:
|
||||
"""List all regions."""
|
||||
result = self.client.table("regions").select("*").execute()
|
||||
return [Region(**r) for r in result.data]
|
||||
|
||||
# ==================== Country Operations ====================
|
||||
|
||||
def create_country(self, country: Country) -> Country:
|
||||
"""Create a new country."""
|
||||
data = {
|
||||
"region_id": str(country.region_id) if country.region_id else None,
|
||||
"name": country.name,
|
||||
"name_local": country.name_local,
|
||||
"code": country.code,
|
||||
"slug": country.slug,
|
||||
"language": country.language,
|
||||
}
|
||||
result = self.client.table("countries").insert(data).execute()
|
||||
return Country(**result.data[0])
|
||||
|
||||
def get_country_by_code(self, code: str) -> Optional[Country]:
|
||||
"""Get country by ISO code."""
|
||||
result = (
|
||||
self.client.table("countries")
|
||||
.select("*")
|
||||
.eq("code", code.upper())
|
||||
.execute()
|
||||
)
|
||||
if result.data:
|
||||
return Country(**result.data[0])
|
||||
return None
|
||||
|
||||
def get_country_by_slug(self, slug: str) -> Optional[Country]:
|
||||
"""Get country by slug."""
|
||||
result = (
|
||||
self.client.table("countries").select("*").eq("slug", slug).execute()
|
||||
)
|
||||
if result.data:
|
||||
return Country(**result.data[0])
|
||||
return None
|
||||
|
||||
def list_countries(self, region_id: Optional[UUID] = None) -> List[Country]:
|
||||
"""List countries, optionally filtered by region."""
|
||||
query = self.client.table("countries").select("*")
|
||||
if region_id:
|
||||
query = query.eq("region_id", str(region_id))
|
||||
result = query.execute()
|
||||
return [Country(**c) for c in result.data]
|
||||
|
||||
# ==================== City Operations ====================
|
||||
|
||||
def create_city(self, city: City) -> City:
|
||||
"""Create a new city."""
|
||||
data = {
|
||||
"country_id": str(city.country_id) if city.country_id else None,
|
||||
"name": city.name,
|
||||
"name_local": city.name_local,
|
||||
"slug": city.slug,
|
||||
"state": city.state,
|
||||
"population": city.population,
|
||||
"latitude": city.latitude,
|
||||
"longitude": city.longitude,
|
||||
}
|
||||
result = self.client.table("cities").insert(data).execute()
|
||||
return City(**result.data[0])
|
||||
|
||||
def get_city_by_slug(
|
||||
self, slug: str, country_id: Optional[UUID] = None
|
||||
) -> Optional[City]:
|
||||
"""Get city by slug, optionally filtered by country."""
|
||||
query = self.client.table("cities").select("*").eq("slug", slug)
|
||||
if country_id:
|
||||
query = query.eq("country_id", str(country_id))
|
||||
result = query.execute()
|
||||
if result.data:
|
||||
return City(**result.data[0])
|
||||
return None
|
||||
|
||||
def list_cities(self, country_id: Optional[UUID] = None) -> List[City]:
|
||||
"""List cities, optionally filtered by country."""
|
||||
query = self.client.table("cities").select("*")
|
||||
if country_id:
|
||||
query = query.eq("country_id", str(country_id))
|
||||
result = query.execute()
|
||||
return [City(**c) for c in result.data]
|
||||
|
||||
# ==================== Tag Operations ====================
|
||||
|
||||
def create_tag(self, tag: Tag) -> Tag:
|
||||
"""Create a new tag."""
|
||||
data = {
|
||||
"category": tag.category.value,
|
||||
"name": tag.name,
|
||||
"slug": tag.slug,
|
||||
"description": tag.description,
|
||||
}
|
||||
result = self.client.table("tags").insert(data).execute()
|
||||
return Tag(**result.data[0])
|
||||
|
||||
def get_tag_by_slug(self, slug: str) -> Optional[Tag]:
|
||||
"""Get tag by slug."""
|
||||
result = self.client.table("tags").select("*").eq("slug", slug).execute()
|
||||
if result.data:
|
||||
return Tag(**result.data[0])
|
||||
return None
|
||||
|
||||
def list_tags(self, category: Optional[TagCategory] = None) -> List[Tag]:
|
||||
"""List tags, optionally filtered by category."""
|
||||
query = self.client.table("tags").select("*")
|
||||
if category:
|
||||
query = query.eq("category", category.value)
|
||||
result = query.execute()
|
||||
return [Tag(**t) for t in result.data]
|
||||
|
||||
def get_or_create_tag(
|
||||
self, name: str, category: TagCategory, description: Optional[str] = None
|
||||
) -> Tag:
|
||||
"""Get existing tag or create new one."""
|
||||
slug = name.lower().replace(" ", "-").replace("&", "and")
|
||||
existing = self.get_tag_by_slug(slug)
|
||||
if existing:
|
||||
return existing
|
||||
return self.create_tag(
|
||||
Tag(category=category, name=name, slug=slug, description=description)
|
||||
)
|
||||
|
||||
# ==================== Company Operations ====================
|
||||
|
||||
def create_company(self, company: Company) -> Company:
|
||||
"""Create a new company."""
|
||||
data = {
|
||||
"city_id": str(company.city_id) if company.city_id else None,
|
||||
"name": company.name,
|
||||
"slug": company.slug,
|
||||
"google_place_id": company.google_place_id,
|
||||
"address": company.address,
|
||||
"phone": company.phone,
|
||||
"website": company.website,
|
||||
"email": company.email,
|
||||
"rating": company.rating,
|
||||
"review_count": company.review_count,
|
||||
"latitude": company.latitude,
|
||||
"longitude": company.longitude,
|
||||
"status": company.status,
|
||||
}
|
||||
result = self.client.table("companies").insert(data).execute()
|
||||
return Company(**result.data[0])
|
||||
|
||||
def update_company(self, company_id: UUID, updates: Dict[str, Any]) -> Company:
|
||||
"""Update a company."""
|
||||
result = (
|
||||
self.client.table("companies")
|
||||
.update(updates)
|
||||
.eq("id", str(company_id))
|
||||
.execute()
|
||||
)
|
||||
return Company(**result.data[0])
|
||||
|
||||
def get_company_by_id(self, company_id: UUID) -> Optional[Company]:
|
||||
"""Get company by ID."""
|
||||
result = (
|
||||
self.client.table("companies")
|
||||
.select("*")
|
||||
.eq("id", str(company_id))
|
||||
.execute()
|
||||
)
|
||||
if result.data:
|
||||
return Company(**result.data[0])
|
||||
return None
|
||||
|
||||
def get_company_by_place_id(self, place_id: str) -> Optional[Company]:
|
||||
"""Get company by Google Place ID (for deduplication)."""
|
||||
result = (
|
||||
self.client.table("companies")
|
||||
.select("*")
|
||||
.eq("google_place_id", place_id)
|
||||
.execute()
|
||||
)
|
||||
if result.data:
|
||||
return Company(**result.data[0])
|
||||
return None
|
||||
|
||||
def list_companies(
|
||||
self,
|
||||
city_id: Optional[UUID] = None,
|
||||
status: Optional[str] = None,
|
||||
limit: int = 100,
|
||||
offset: int = 0,
|
||||
) -> List[Company]:
|
||||
"""List companies with optional filters."""
|
||||
query = self.client.table("companies").select("*")
|
||||
if city_id:
|
||||
query = query.eq("city_id", str(city_id))
|
||||
if status:
|
||||
query = query.eq("status", status)
|
||||
query = query.range(offset, offset + limit - 1)
|
||||
result = query.execute()
|
||||
return [Company(**c) for c in result.data]
|
||||
|
||||
def count_companies(
|
||||
self, city_id: Optional[UUID] = None, status: Optional[str] = None
|
||||
) -> int:
|
||||
"""Count companies with optional filters."""
|
||||
query = self.client.table("companies").select("id", count="exact")
|
||||
if city_id:
|
||||
query = query.eq("city_id", str(city_id))
|
||||
if status:
|
||||
query = query.eq("status", status)
|
||||
result = query.execute()
|
||||
return result.count or 0
|
||||
|
||||
# ==================== Company Tag Operations ====================
|
||||
|
||||
def add_company_tag(
|
||||
self, company_id: UUID, tag_id: UUID, confidence: float = 1.0
|
||||
) -> CompanyTag:
|
||||
"""Add a tag to a company."""
|
||||
data = {
|
||||
"company_id": str(company_id),
|
||||
"tag_id": str(tag_id),
|
||||
"confidence": confidence,
|
||||
}
|
||||
result = self.client.table("company_tags").insert(data).execute()
|
||||
return CompanyTag(**result.data[0])
|
||||
|
||||
def get_company_tags(self, company_id: UUID) -> List[Tag]:
|
||||
"""Get all tags for a company."""
|
||||
result = (
|
||||
self.client.table("company_tags")
|
||||
.select("tag_id, tags(*)")
|
||||
.eq("company_id", str(company_id))
|
||||
.execute()
|
||||
)
|
||||
tags = []
|
||||
for item in result.data:
|
||||
if item.get("tags"):
|
||||
tags.append(Tag(**item["tags"]))
|
||||
return tags
|
||||
|
||||
def remove_company_tags(self, company_id: UUID) -> None:
|
||||
"""Remove all tags from a company."""
|
||||
self.client.table("company_tags").delete().eq(
|
||||
"company_id", str(company_id)
|
||||
).execute()
|
||||
|
||||
# ==================== City Tag Aggregation ====================
|
||||
|
||||
def upsert_city_tag(self, city_id: UUID, tag_id: UUID, count: int) -> CityTag:
|
||||
"""Upsert city tag aggregation."""
|
||||
data = {
|
||||
"city_id": str(city_id),
|
||||
"tag_id": str(tag_id),
|
||||
"company_count": count,
|
||||
}
|
||||
result = (
|
||||
self.client.table("city_tags")
|
||||
.upsert(data, on_conflict="city_id,tag_id")
|
||||
.execute()
|
||||
)
|
||||
return CityTag(**result.data[0])
|
||||
|
||||
def get_city_tags(self, city_id: UUID) -> List[Dict[str, Any]]:
|
||||
"""Get all tag aggregations for a city with tag details."""
|
||||
result = (
|
||||
self.client.table("city_tags")
|
||||
.select("*, tags(*)")
|
||||
.eq("city_id", str(city_id))
|
||||
.execute()
|
||||
)
|
||||
return result.data
|
||||
|
||||
# ==================== Country Tag Aggregation ====================
|
||||
|
||||
def upsert_country_tag(
|
||||
self, country_id: UUID, tag_id: UUID, company_count: int, city_count: int
|
||||
) -> CountryTag:
|
||||
"""Upsert country tag aggregation."""
|
||||
data = {
|
||||
"country_id": str(country_id),
|
||||
"tag_id": str(tag_id),
|
||||
"company_count": company_count,
|
||||
"city_count": city_count,
|
||||
}
|
||||
result = (
|
||||
self.client.table("country_tags")
|
||||
.upsert(data, on_conflict="country_id,tag_id")
|
||||
.execute()
|
||||
)
|
||||
return CountryTag(**result.data[0])
|
||||
|
||||
def get_country_tags(self, country_id: UUID) -> List[Dict[str, Any]]:
|
||||
"""Get all tag aggregations for a country with tag details."""
|
||||
result = (
|
||||
self.client.table("country_tags")
|
||||
.select("*, tags(*)")
|
||||
.eq("country_id", str(country_id))
|
||||
.execute()
|
||||
)
|
||||
return result.data
|
||||
|
||||
# ==================== Execution Plan Operations ====================
|
||||
|
||||
def create_execution_plan(self, plan: ExecutionPlan) -> ExecutionPlan:
|
||||
"""Create a new execution plan."""
|
||||
data = {
|
||||
"name": plan.name,
|
||||
"country_code": plan.country_code,
|
||||
"cities": plan.cities,
|
||||
"status": plan.status.value,
|
||||
}
|
||||
result = self.client.table("execution_plans").insert(data).execute()
|
||||
return ExecutionPlan(**result.data[0])
|
||||
|
||||
def update_execution_plan(
|
||||
self, plan_id: UUID, updates: Dict[str, Any]
|
||||
) -> ExecutionPlan:
|
||||
"""Update an execution plan."""
|
||||
result = (
|
||||
self.client.table("execution_plans")
|
||||
.update(updates)
|
||||
.eq("id", str(plan_id))
|
||||
.execute()
|
||||
)
|
||||
return ExecutionPlan(**result.data[0])
|
||||
|
||||
def get_execution_plan(self, plan_id: UUID) -> Optional[ExecutionPlan]:
|
||||
"""Get execution plan by ID."""
|
||||
result = (
|
||||
self.client.table("execution_plans")
|
||||
.select("*")
|
||||
.eq("id", str(plan_id))
|
||||
.execute()
|
||||
)
|
||||
if result.data:
|
||||
plan = ExecutionPlan(**result.data[0])
|
||||
# Load steps
|
||||
steps_result = (
|
||||
self.client.table("plan_steps")
|
||||
.select("*")
|
||||
.eq("plan_id", str(plan_id))
|
||||
.execute()
|
||||
)
|
||||
plan.steps = [PlanStep(**s) for s in steps_result.data]
|
||||
return plan
|
||||
return None
|
||||
|
||||
def list_execution_plans(
|
||||
self, status: Optional[StepStatus] = None
|
||||
) -> List[ExecutionPlan]:
|
||||
"""List execution plans."""
|
||||
query = self.client.table("execution_plans").select("*")
|
||||
if status:
|
||||
query = query.eq("status", status.value)
|
||||
result = query.execute()
|
||||
return [ExecutionPlan(**p) for p in result.data]
|
||||
|
||||
# ==================== Plan Step Operations ====================
|
||||
|
||||
def create_plan_step(self, step: PlanStep) -> PlanStep:
|
||||
"""Create a new plan step."""
|
||||
data = {
|
||||
"plan_id": str(step.plan_id) if step.plan_id else None,
|
||||
"step_name": step.step_name,
|
||||
"city_id": str(step.city_id) if step.city_id else None,
|
||||
"status": step.status.value,
|
||||
"items_processed": step.items_processed,
|
||||
"items_total": step.items_total,
|
||||
}
|
||||
result = self.client.table("plan_steps").insert(data).execute()
|
||||
return PlanStep(**result.data[0])
|
||||
|
||||
def update_plan_step(self, step_id: UUID, updates: Dict[str, Any]) -> PlanStep:
|
||||
"""Update a plan step."""
|
||||
result = (
|
||||
self.client.table("plan_steps")
|
||||
.update(updates)
|
||||
.eq("id", str(step_id))
|
||||
.execute()
|
||||
)
|
||||
return PlanStep(**result.data[0])
|
||||
|
||||
def get_plan_steps(
|
||||
self, plan_id: UUID, step_name: Optional[str] = None
|
||||
) -> List[PlanStep]:
|
||||
"""Get steps for a plan."""
|
||||
query = (
|
||||
self.client.table("plan_steps").select("*").eq("plan_id", str(plan_id))
|
||||
)
|
||||
if step_name:
|
||||
query = query.eq("step_name", step_name)
|
||||
result = query.execute()
|
||||
return [PlanStep(**s) for s in result.data]
|
||||
|
||||
# ==================== Error Logging ====================
|
||||
|
||||
def log_error(
|
||||
self,
|
||||
entity_type: str,
|
||||
entity_id: Optional[UUID],
|
||||
operation: str,
|
||||
error_message: str,
|
||||
details: Optional[Dict[str, Any]] = None,
|
||||
) -> None:
|
||||
"""Log an error to the database."""
|
||||
data = {
|
||||
"entity_type": entity_type,
|
||||
"entity_id": str(entity_id) if entity_id else None,
|
||||
"operation": operation,
|
||||
"error_message": error_message,
|
||||
"details": details or {},
|
||||
}
|
||||
try:
|
||||
self.client.table("error_logs").insert(data).execute()
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to log error to database: {e}")
|
||||
@@ -0,0 +1,309 @@
|
||||
"""
|
||||
Tag aggregation service for cities and countries.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from typing import Dict, List, Any
|
||||
from uuid import UUID
|
||||
from collections import defaultdict
|
||||
|
||||
from .supabase import SupabaseService
|
||||
from ..models.schemas import TagCategory
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class TagAggregationService:
|
||||
"""Service for aggregating tags across cities and countries."""
|
||||
|
||||
def __init__(self, db: SupabaseService = None):
|
||||
"""
|
||||
Initialize tag aggregation service.
|
||||
|
||||
Args:
|
||||
db: Supabase service instance (creates new one if not provided)
|
||||
"""
|
||||
self.db = db or SupabaseService()
|
||||
|
||||
def aggregate_city_tags(self, city_id: UUID) -> Dict[str, int]:
|
||||
"""
|
||||
Aggregate all company tags for a city.
|
||||
|
||||
Counts how many companies have each tag and stores in city_tags table.
|
||||
|
||||
Args:
|
||||
city_id: City UUID
|
||||
|
||||
Returns:
|
||||
Dictionary mapping tag slugs to company counts
|
||||
"""
|
||||
logger.info(f"Aggregating tags for city {city_id}")
|
||||
|
||||
# Get all companies in the city
|
||||
companies = self.db.list_companies(city_id=city_id, limit=1000)
|
||||
|
||||
# Count tags across all companies
|
||||
tag_counts = defaultdict(int)
|
||||
tag_id_map = {}
|
||||
|
||||
for company in companies:
|
||||
company_tags = self.db.get_company_tags(company.id)
|
||||
for tag in company_tags:
|
||||
tag_counts[tag.slug] += 1
|
||||
tag_id_map[tag.slug] = tag.id
|
||||
|
||||
# Update city_tags table
|
||||
for slug, count in tag_counts.items():
|
||||
tag_id = tag_id_map[slug]
|
||||
self.db.upsert_city_tag(city_id, tag_id, count)
|
||||
|
||||
logger.info(f"Aggregated {len(tag_counts)} tags for city {city_id}")
|
||||
return dict(tag_counts)
|
||||
|
||||
def aggregate_country_tags(self, country_id: UUID) -> Dict[str, Dict[str, int]]:
|
||||
"""
|
||||
Aggregate all city tags for a country.
|
||||
|
||||
Rolls up city-level tag counts to country level.
|
||||
|
||||
Args:
|
||||
country_id: Country UUID
|
||||
|
||||
Returns:
|
||||
Dictionary mapping tag slugs to {company_count, city_count}
|
||||
"""
|
||||
logger.info(f"Aggregating tags for country {country_id}")
|
||||
|
||||
# Get all cities in the country
|
||||
cities = self.db.list_cities(country_id=country_id)
|
||||
|
||||
# Aggregate across all cities
|
||||
tag_stats = defaultdict(lambda: {"company_count": 0, "city_count": 0})
|
||||
tag_id_map = {}
|
||||
|
||||
for city in cities:
|
||||
city_tags = self.db.get_city_tags(city.id)
|
||||
|
||||
for ct in city_tags:
|
||||
tag_data = ct.get("tags", {})
|
||||
tag_slug = tag_data.get("slug")
|
||||
tag_id = tag_data.get("id")
|
||||
|
||||
if tag_slug and tag_id:
|
||||
company_count = ct.get("company_count", 0)
|
||||
tag_stats[tag_slug]["company_count"] += company_count
|
||||
tag_stats[tag_slug]["city_count"] += 1
|
||||
tag_id_map[tag_slug] = tag_id
|
||||
|
||||
# Update country_tags table
|
||||
for slug, stats in tag_stats.items():
|
||||
tag_id = tag_id_map[slug]
|
||||
self.db.upsert_country_tag(
|
||||
country_id, tag_id, stats["company_count"], stats["city_count"]
|
||||
)
|
||||
|
||||
logger.info(f"Aggregated {len(tag_stats)} tags for country {country_id}")
|
||||
return dict(tag_stats)
|
||||
|
||||
def get_city_tags_by_category(
|
||||
self, city_id: UUID
|
||||
) -> Dict[str, List[Dict[str, Any]]]:
|
||||
"""
|
||||
Get city tags organized by category.
|
||||
|
||||
Args:
|
||||
city_id: City UUID
|
||||
|
||||
Returns:
|
||||
Dictionary with categories as keys and lists of tag info
|
||||
"""
|
||||
city_tags = self.db.get_city_tags(city_id)
|
||||
|
||||
result = {
|
||||
"services": [],
|
||||
"industries": [],
|
||||
"modules": [],
|
||||
"partner_levels": [],
|
||||
}
|
||||
|
||||
for ct in city_tags:
|
||||
tag_data = ct.get("tags", {})
|
||||
category = tag_data.get("category")
|
||||
company_count = ct.get("company_count", 0)
|
||||
|
||||
tag_info = {
|
||||
"name": tag_data.get("name"),
|
||||
"slug": tag_data.get("slug"),
|
||||
"count": company_count,
|
||||
}
|
||||
|
||||
if category == "service":
|
||||
result["services"].append(tag_info)
|
||||
elif category == "industry":
|
||||
result["industries"].append(tag_info)
|
||||
elif category == "module":
|
||||
result["modules"].append(tag_info)
|
||||
elif category == "partner_level":
|
||||
result["partner_levels"].append(tag_info)
|
||||
|
||||
# Sort each category by count (descending)
|
||||
for key in result:
|
||||
result[key].sort(key=lambda x: x["count"], reverse=True)
|
||||
|
||||
return result
|
||||
|
||||
def get_country_tags_by_category(
|
||||
self, country_id: UUID
|
||||
) -> Dict[str, List[Dict[str, Any]]]:
|
||||
"""
|
||||
Get country tags organized by category.
|
||||
|
||||
Args:
|
||||
country_id: Country UUID
|
||||
|
||||
Returns:
|
||||
Dictionary with categories as keys and lists of tag info
|
||||
"""
|
||||
country_tags = self.db.get_country_tags(country_id)
|
||||
|
||||
result = {
|
||||
"services": [],
|
||||
"industries": [],
|
||||
"modules": [],
|
||||
"partner_levels": [],
|
||||
}
|
||||
|
||||
for ct in country_tags:
|
||||
tag_data = ct.get("tags", {})
|
||||
category = tag_data.get("category")
|
||||
|
||||
tag_info = {
|
||||
"name": tag_data.get("name"),
|
||||
"slug": tag_data.get("slug"),
|
||||
"company_count": ct.get("company_count", 0),
|
||||
"city_count": ct.get("city_count", 0),
|
||||
}
|
||||
|
||||
if category == "service":
|
||||
result["services"].append(tag_info)
|
||||
elif category == "industry":
|
||||
result["industries"].append(tag_info)
|
||||
elif category == "module":
|
||||
result["modules"].append(tag_info)
|
||||
elif category == "partner_level":
|
||||
result["partner_levels"].append(tag_info)
|
||||
|
||||
# Sort each category by company count (descending)
|
||||
for key in result:
|
||||
result[key].sort(key=lambda x: x["company_count"], reverse=True)
|
||||
|
||||
return result
|
||||
|
||||
def export_city_tags_json(self, city_id: UUID) -> Dict[str, Any]:
|
||||
"""
|
||||
Export city tags as JSON for frontend filtering.
|
||||
|
||||
Args:
|
||||
city_id: City UUID
|
||||
|
||||
Returns:
|
||||
JSON-serializable dictionary
|
||||
"""
|
||||
tags_by_category = self.get_city_tags_by_category(city_id)
|
||||
|
||||
return {
|
||||
"filters": {
|
||||
"services": [
|
||||
{"value": t["slug"], "label": t["name"], "count": t["count"]}
|
||||
for t in tags_by_category["services"]
|
||||
],
|
||||
"industries": [
|
||||
{"value": t["slug"], "label": t["name"], "count": t["count"]}
|
||||
for t in tags_by_category["industries"]
|
||||
],
|
||||
"modules": [
|
||||
{"value": t["slug"], "label": t["name"], "count": t["count"]}
|
||||
for t in tags_by_category["modules"]
|
||||
],
|
||||
"partner_levels": [
|
||||
{"value": t["slug"], "label": t["name"], "count": t["count"]}
|
||||
for t in tags_by_category["partner_levels"]
|
||||
],
|
||||
}
|
||||
}
|
||||
|
||||
def export_country_tags_json(self, country_id: UUID) -> Dict[str, Any]:
|
||||
"""
|
||||
Export country tags as JSON for frontend filtering.
|
||||
|
||||
Args:
|
||||
country_id: Country UUID
|
||||
|
||||
Returns:
|
||||
JSON-serializable dictionary
|
||||
"""
|
||||
tags_by_category = self.get_country_tags_by_category(country_id)
|
||||
|
||||
return {
|
||||
"filters": {
|
||||
"services": [
|
||||
{
|
||||
"value": t["slug"],
|
||||
"label": t["name"],
|
||||
"company_count": t["company_count"],
|
||||
"city_count": t["city_count"],
|
||||
}
|
||||
for t in tags_by_category["services"]
|
||||
],
|
||||
"industries": [
|
||||
{
|
||||
"value": t["slug"],
|
||||
"label": t["name"],
|
||||
"company_count": t["company_count"],
|
||||
"city_count": t["city_count"],
|
||||
}
|
||||
for t in tags_by_category["industries"]
|
||||
],
|
||||
"modules": [
|
||||
{
|
||||
"value": t["slug"],
|
||||
"label": t["name"],
|
||||
"company_count": t["company_count"],
|
||||
"city_count": t["city_count"],
|
||||
}
|
||||
for t in tags_by_category["modules"]
|
||||
],
|
||||
"partner_levels": [
|
||||
{
|
||||
"value": t["slug"],
|
||||
"label": t["name"],
|
||||
"company_count": t["company_count"],
|
||||
"city_count": t["city_count"],
|
||||
}
|
||||
for t in tags_by_category["partner_levels"]
|
||||
],
|
||||
}
|
||||
}
|
||||
|
||||
def refresh_all_aggregations(self, country_id: UUID) -> None:
|
||||
"""
|
||||
Refresh all tag aggregations for a country.
|
||||
|
||||
First aggregates all city tags, then rolls up to country level.
|
||||
|
||||
Args:
|
||||
country_id: Country UUID
|
||||
"""
|
||||
logger.info(f"Refreshing all aggregations for country {country_id}")
|
||||
|
||||
# Get all cities
|
||||
cities = self.db.list_cities(country_id=country_id)
|
||||
|
||||
# Aggregate each city
|
||||
for city in cities:
|
||||
self.aggregate_city_tags(city.id)
|
||||
|
||||
# Aggregate country
|
||||
self.aggregate_country_tags(country_id)
|
||||
|
||||
logger.info(f"Completed aggregation refresh for country {country_id}")
|
||||
@@ -0,0 +1 @@
|
||||
# Templates directory - Jinja2 templates are auto-generated on first run
|
||||
@@ -0,0 +1,6 @@
|
||||
"""Utility modules for Odoo Directory."""
|
||||
|
||||
from .logger import get_logger, setup_logging
|
||||
from .retry import retry_with_backoff
|
||||
|
||||
__all__ = ["get_logger", "setup_logging", "retry_with_backoff"]
|
||||
@@ -0,0 +1,53 @@
|
||||
"""
|
||||
Logging configuration for Odoo Directory CLI.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import sys
|
||||
from typing import Optional
|
||||
|
||||
from rich.console import Console
|
||||
from rich.logging import RichHandler
|
||||
|
||||
# Global console for rich output
|
||||
console = Console()
|
||||
|
||||
|
||||
def setup_logging(level: str = "INFO") -> None:
|
||||
"""
|
||||
Configure logging with rich handler for pretty console output.
|
||||
|
||||
Args:
|
||||
level: Log level (DEBUG, INFO, WARNING, ERROR, CRITICAL)
|
||||
"""
|
||||
logging.basicConfig(
|
||||
level=level.upper(),
|
||||
format="%(message)s",
|
||||
datefmt="[%X]",
|
||||
handlers=[
|
||||
RichHandler(
|
||||
console=console,
|
||||
show_time=True,
|
||||
show_path=False,
|
||||
markup=True,
|
||||
rich_tracebacks=True,
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def get_logger(name: str, level: Optional[str] = None) -> logging.Logger:
|
||||
"""
|
||||
Get a logger instance with optional level override.
|
||||
|
||||
Args:
|
||||
name: Logger name (usually __name__)
|
||||
level: Optional level override
|
||||
|
||||
Returns:
|
||||
Configured logger instance
|
||||
"""
|
||||
logger = logging.getLogger(name)
|
||||
if level:
|
||||
logger.setLevel(level.upper())
|
||||
return logger
|
||||
@@ -0,0 +1,92 @@
|
||||
"""
|
||||
Retry utilities with exponential backoff.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
from functools import wraps
|
||||
from typing import Callable, TypeVar, Any, Tuple, Type
|
||||
import logging
|
||||
|
||||
from tenacity import (
|
||||
retry,
|
||||
stop_after_attempt,
|
||||
wait_exponential,
|
||||
retry_if_exception_type,
|
||||
before_sleep_log,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
T = TypeVar("T")
|
||||
|
||||
|
||||
def retry_with_backoff(
|
||||
max_attempts: int = 3,
|
||||
min_wait: float = 1.0,
|
||||
max_wait: float = 60.0,
|
||||
exceptions: Tuple[Type[Exception], ...] = (Exception,),
|
||||
) -> Callable:
|
||||
"""
|
||||
Decorator for retrying functions with exponential backoff.
|
||||
|
||||
Args:
|
||||
max_attempts: Maximum number of retry attempts
|
||||
min_wait: Minimum wait time between retries (seconds)
|
||||
max_wait: Maximum wait time between retries (seconds)
|
||||
exceptions: Tuple of exception types to retry on
|
||||
|
||||
Returns:
|
||||
Decorated function with retry logic
|
||||
"""
|
||||
return retry(
|
||||
stop=stop_after_attempt(max_attempts),
|
||||
wait=wait_exponential(multiplier=1, min=min_wait, max=max_wait),
|
||||
retry=retry_if_exception_type(exceptions),
|
||||
before_sleep=before_sleep_log(logger, logging.WARNING),
|
||||
reraise=True,
|
||||
)
|
||||
|
||||
|
||||
async def async_retry_with_backoff(
|
||||
func: Callable[..., T],
|
||||
*args,
|
||||
max_attempts: int = 3,
|
||||
min_wait: float = 1.0,
|
||||
max_wait: float = 60.0,
|
||||
exceptions: Tuple[Type[Exception], ...] = (Exception,),
|
||||
**kwargs,
|
||||
) -> T:
|
||||
"""
|
||||
Async function with retry logic and exponential backoff.
|
||||
|
||||
Args:
|
||||
func: Async function to call
|
||||
*args: Positional arguments for func
|
||||
max_attempts: Maximum number of retry attempts
|
||||
min_wait: Minimum wait time between retries (seconds)
|
||||
max_wait: Maximum wait time between retries (seconds)
|
||||
exceptions: Tuple of exception types to retry on
|
||||
**kwargs: Keyword arguments for func
|
||||
|
||||
Returns:
|
||||
Result of the function call
|
||||
"""
|
||||
last_exception = None
|
||||
wait_time = min_wait
|
||||
|
||||
for attempt in range(1, max_attempts + 1):
|
||||
try:
|
||||
return await func(*args, **kwargs)
|
||||
except exceptions as e:
|
||||
last_exception = e
|
||||
if attempt < max_attempts:
|
||||
logger.warning(
|
||||
f"Attempt {attempt}/{max_attempts} failed: {e}. "
|
||||
f"Retrying in {wait_time:.1f}s..."
|
||||
)
|
||||
await asyncio.sleep(wait_time)
|
||||
wait_time = min(wait_time * 2, max_wait)
|
||||
else:
|
||||
logger.error(f"All {max_attempts} attempts failed: {e}")
|
||||
|
||||
raise last_exception
|
||||
Reference in New Issue
Block a user