new site
This commit is contained in:
@@ -0,0 +1,15 @@
|
||||
"""CLI commands for Odoo Directory."""
|
||||
|
||||
from .setup import setup_command
|
||||
from .plan import plan_command
|
||||
from .execute import execute_command
|
||||
from .export import export_command
|
||||
from .deploy import deploy_command
|
||||
|
||||
__all__ = [
|
||||
"setup_command",
|
||||
"plan_command",
|
||||
"execute_command",
|
||||
"export_command",
|
||||
"deploy_command",
|
||||
]
|
||||
@@ -0,0 +1,148 @@
|
||||
"""
|
||||
Deploy command for pushing changes to Git.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from datetime import datetime
|
||||
from typing import Optional
|
||||
|
||||
import typer
|
||||
from rich.console import Console
|
||||
|
||||
from ..config import get_settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
console = Console()
|
||||
|
||||
|
||||
def deploy_command(
|
||||
message: Optional[str] = None,
|
||||
push: bool = True,
|
||||
) -> None:
|
||||
"""
|
||||
Deploy exported files via Git.
|
||||
|
||||
Adds all changes in the export directory, commits them,
|
||||
and optionally pushes to the remote repository.
|
||||
|
||||
Args:
|
||||
message: Custom commit message (auto-generated if not provided)
|
||||
push: Whether to push to remote after commit
|
||||
"""
|
||||
console.print("\n[bold blue]Deploying changes[/bold blue]\n")
|
||||
|
||||
settings = get_settings()
|
||||
project_root = Path(settings.export_dir).parent
|
||||
|
||||
# Verify we're in a git repo
|
||||
if not (project_root / ".git").exists():
|
||||
console.print("[red]Not a git repository![/red]")
|
||||
console.print(f"Expected .git directory in {project_root}")
|
||||
raise typer.Exit(1)
|
||||
|
||||
try:
|
||||
# Check for changes
|
||||
result = subprocess.run(
|
||||
["git", "status", "--porcelain"],
|
||||
cwd=project_root,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
)
|
||||
|
||||
if not result.stdout.strip():
|
||||
console.print("[yellow]No changes to deploy.[/yellow]")
|
||||
return
|
||||
|
||||
# Show changes
|
||||
console.print("[bold]Changes to deploy:[/bold]")
|
||||
console.print(result.stdout)
|
||||
|
||||
# Confirm deployment
|
||||
if not typer.confirm("Deploy these changes?"):
|
||||
console.print("[yellow]Deployment cancelled.[/yellow]")
|
||||
raise typer.Exit(0)
|
||||
|
||||
# Generate commit message if not provided
|
||||
if not message:
|
||||
timestamp = datetime.now().strftime("%Y-%m-%d %H:%M")
|
||||
message = f"Directory update: {timestamp}"
|
||||
|
||||
# Stage changes
|
||||
console.print("Staging changes...")
|
||||
subprocess.run(
|
||||
["git", "add", "export/", "static/images/companies/"],
|
||||
cwd=project_root,
|
||||
check=True,
|
||||
)
|
||||
|
||||
# Commit
|
||||
console.print(f"Committing: {message}")
|
||||
subprocess.run(
|
||||
["git", "commit", "-m", message],
|
||||
cwd=project_root,
|
||||
check=True,
|
||||
)
|
||||
|
||||
# Push if requested
|
||||
if push:
|
||||
console.print("Pushing to remote...")
|
||||
subprocess.run(
|
||||
["git", "push"],
|
||||
cwd=project_root,
|
||||
check=True,
|
||||
)
|
||||
console.print("[green]Changes pushed to remote.[/green]")
|
||||
else:
|
||||
console.print("[yellow]Changes committed locally but not pushed.[/yellow]")
|
||||
|
||||
console.print("\n[bold green]Deployment complete![/bold green]")
|
||||
|
||||
# Show deployment info
|
||||
console.print("\nVercel will automatically deploy the changes.")
|
||||
console.print("Check deployment status at: https://vercel.com/dashboard")
|
||||
|
||||
except subprocess.CalledProcessError as e:
|
||||
console.print(f"[red]Git command failed: {e}[/red]")
|
||||
if e.stderr:
|
||||
console.print(f"Error: {e.stderr}")
|
||||
raise typer.Exit(1)
|
||||
|
||||
except Exception as e:
|
||||
console.print(f"[red]Deployment failed: {e}[/red]")
|
||||
logger.exception("Deployment error")
|
||||
raise typer.Exit(1)
|
||||
|
||||
|
||||
def status_command() -> None:
|
||||
"""Show git status of the project."""
|
||||
settings = get_settings()
|
||||
project_root = Path(settings.export_dir).parent
|
||||
|
||||
try:
|
||||
# Git status
|
||||
result = subprocess.run(
|
||||
["git", "status"],
|
||||
cwd=project_root,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
)
|
||||
console.print(result.stdout)
|
||||
|
||||
# Last commit
|
||||
console.print("\n[bold]Last commit:[/bold]")
|
||||
result = subprocess.run(
|
||||
["git", "log", "-1", "--oneline"],
|
||||
cwd=project_root,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
)
|
||||
console.print(result.stdout)
|
||||
|
||||
except subprocess.CalledProcessError as e:
|
||||
console.print(f"[red]Git command failed: {e}[/red]")
|
||||
raise typer.Exit(1)
|
||||
@@ -0,0 +1,341 @@
|
||||
"""
|
||||
Execute command for running pipeline steps.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from typing import Optional, List
|
||||
from uuid import UUID
|
||||
import re
|
||||
|
||||
import typer
|
||||
from rich.console import Console
|
||||
from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn
|
||||
|
||||
from ..services.supabase import SupabaseService
|
||||
from ..services.apify import ApifyService
|
||||
from ..services.firecrawl import FirecrawlService
|
||||
from ..services.haiku import HaikuService
|
||||
from ..services.images import ImageService
|
||||
from ..services.tags import TagAggregationService
|
||||
from ..models.schemas import Company, StepStatus, TagCategory
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
console = Console()
|
||||
|
||||
|
||||
def execute_command(
|
||||
plan_id: str,
|
||||
step: Optional[str] = None,
|
||||
city: Optional[str] = None,
|
||||
dry_run: bool = False,
|
||||
) -> None:
|
||||
"""
|
||||
Execute an execution plan or specific step.
|
||||
|
||||
Runs the pipeline to scrape, enrich, and generate content for
|
||||
Odoo partners.
|
||||
|
||||
Args:
|
||||
plan_id: UUID of the execution plan
|
||||
step: Specific step to run (scrape, enrich, content, export)
|
||||
city: Specific city slug to process
|
||||
dry_run: If True, don't make actual API calls
|
||||
"""
|
||||
console.print(f"\n[bold blue]Executing Plan: {plan_id}[/bold blue]\n")
|
||||
|
||||
db = SupabaseService()
|
||||
|
||||
try:
|
||||
plan_uuid = UUID(plan_id)
|
||||
except ValueError:
|
||||
console.print(f"[red]Invalid plan ID: {plan_id}[/red]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
plan = db.get_execution_plan(plan_uuid)
|
||||
if not plan:
|
||||
console.print(f"[red]Plan not found: {plan_id}[/red]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
# Get country info
|
||||
country = db.get_country_by_code(plan.country_code)
|
||||
if not country:
|
||||
console.print(f"[red]Country not found: {plan.country_code}[/red]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
# Filter cities if specified
|
||||
city_slugs = plan.cities
|
||||
if city:
|
||||
if city not in city_slugs:
|
||||
console.print(f"[red]City {city} not in plan[/red]")
|
||||
raise typer.Exit(1)
|
||||
city_slugs = [city]
|
||||
|
||||
# Determine steps to run
|
||||
if step:
|
||||
steps_to_run = [step]
|
||||
else:
|
||||
steps_to_run = ["scrape", "enrich", "content"]
|
||||
|
||||
console.print(f"Country: {country.name}")
|
||||
console.print(f"Cities: {len(city_slugs)}")
|
||||
console.print(f"Steps: {', '.join(steps_to_run)}")
|
||||
console.print(f"Dry run: {dry_run}")
|
||||
console.print()
|
||||
|
||||
# Update plan status
|
||||
if not dry_run:
|
||||
db.update_execution_plan(
|
||||
plan_uuid, {"status": "running", "started_at": datetime.utcnow().isoformat()}
|
||||
)
|
||||
|
||||
# Run each step
|
||||
for step_name in steps_to_run:
|
||||
console.print(f"\n[bold cyan]Running step: {step_name}[/bold cyan]\n")
|
||||
|
||||
for city_slug in city_slugs:
|
||||
city_obj = db.get_city_by_slug(city_slug, country.id)
|
||||
if not city_obj:
|
||||
console.print(f"[yellow]City not found: {city_slug}[/yellow]")
|
||||
continue
|
||||
|
||||
try:
|
||||
if step_name == "scrape":
|
||||
_run_scrape_step(db, city_obj, country, dry_run)
|
||||
elif step_name == "enrich":
|
||||
_run_enrich_step(db, city_obj, dry_run)
|
||||
elif step_name == "content":
|
||||
_run_content_step(db, city_obj, country, dry_run)
|
||||
except Exception as e:
|
||||
console.print(f"[red]Error in {step_name} for {city_slug}: {e}[/red]")
|
||||
logger.exception(f"Step {step_name} failed for {city_slug}")
|
||||
|
||||
# Update plan status
|
||||
if not dry_run:
|
||||
db.update_execution_plan(
|
||||
plan_uuid, {"status": "completed", "completed_at": datetime.utcnow().isoformat()}
|
||||
)
|
||||
|
||||
console.print("\n[bold green]Execution complete![/bold green]")
|
||||
|
||||
|
||||
def _run_scrape_step(db: SupabaseService, city, country, dry_run: bool) -> None:
|
||||
"""Run the scrape step for a city."""
|
||||
console.print(f"Scraping {city.name}...")
|
||||
|
||||
if dry_run:
|
||||
console.print("[yellow]Dry run - skipping actual scrape[/yellow]")
|
||||
return
|
||||
|
||||
apify = ApifyService()
|
||||
|
||||
# Scrape Google Maps
|
||||
results = apify.scrape_city(
|
||||
city_name=city.name,
|
||||
language=country.language,
|
||||
country=country.code,
|
||||
)
|
||||
|
||||
# Deduplicate
|
||||
results = apify.deduplicate_results(results)
|
||||
console.print(f"Found {len(results)} unique companies")
|
||||
|
||||
# Save to database
|
||||
saved = 0
|
||||
for result in results:
|
||||
# Check if already exists
|
||||
if result.placeId:
|
||||
existing = db.get_company_by_place_id(result.placeId)
|
||||
if existing:
|
||||
continue
|
||||
|
||||
# Create slug from name
|
||||
slug = _create_slug(result.title)
|
||||
|
||||
# Extract location
|
||||
lat = lng = None
|
||||
if result.location:
|
||||
lat = result.location.get("lat")
|
||||
lng = result.location.get("lng")
|
||||
|
||||
company = Company(
|
||||
city_id=city.id,
|
||||
name=result.title,
|
||||
slug=slug,
|
||||
google_place_id=result.placeId,
|
||||
address=result.address,
|
||||
phone=result.phone,
|
||||
website=result.website,
|
||||
rating=result.rating,
|
||||
review_count=result.reviewsCount,
|
||||
latitude=lat,
|
||||
longitude=lng,
|
||||
status="scraped",
|
||||
last_scraped_at=datetime.utcnow(),
|
||||
)
|
||||
|
||||
db.create_company(company)
|
||||
saved += 1
|
||||
|
||||
console.print(f"[green]Saved {saved} new companies[/green]")
|
||||
|
||||
|
||||
def _run_enrich_step(db: SupabaseService, city, dry_run: bool) -> None:
|
||||
"""Run the enrich step for companies in a city."""
|
||||
console.print(f"Enriching companies in {city.name}...")
|
||||
|
||||
# Get companies that need enrichment
|
||||
companies = db.list_companies(city_id=city.id, status="scraped", limit=100)
|
||||
console.print(f"Found {len(companies)} companies to enrich")
|
||||
|
||||
if not companies or dry_run:
|
||||
if dry_run:
|
||||
console.print("[yellow]Dry run - skipping enrichment[/yellow]")
|
||||
return
|
||||
|
||||
firecrawl = FirecrawlService()
|
||||
|
||||
with Progress(
|
||||
SpinnerColumn(),
|
||||
TextColumn("[progress.description]{task.description}"),
|
||||
BarColumn(),
|
||||
TextColumn("[progress.percentage]{task.percentage:>3.0f}%"),
|
||||
console=console,
|
||||
) as progress:
|
||||
task = progress.add_task("Enriching...", total=len(companies))
|
||||
|
||||
for company in companies:
|
||||
if not company.website:
|
||||
progress.advance(task)
|
||||
continue
|
||||
|
||||
try:
|
||||
info = firecrawl.extract_company_info(company.website)
|
||||
|
||||
updates = {
|
||||
"website_markdown": info.get("markdown"),
|
||||
"logo_url": info.get("logo_url"),
|
||||
"email": info.get("email") or company.email,
|
||||
"status": "enriched",
|
||||
"last_enriched_at": datetime.utcnow().isoformat(),
|
||||
}
|
||||
|
||||
db.update_company(company.id, updates)
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to enrich {company.name}: {e}")
|
||||
|
||||
progress.advance(task)
|
||||
|
||||
console.print("[green]Enrichment complete[/green]")
|
||||
|
||||
|
||||
def _run_content_step(db: SupabaseService, city, country, dry_run: bool) -> None:
|
||||
"""Run the content generation step."""
|
||||
console.print(f"Generating content for {city.name}...")
|
||||
|
||||
# Get companies that need content
|
||||
companies = db.list_companies(city_id=city.id, status="enriched", limit=100)
|
||||
console.print(f"Found {len(companies)} companies for content generation")
|
||||
|
||||
if not companies or dry_run:
|
||||
if dry_run:
|
||||
console.print("[yellow]Dry run - skipping content generation[/yellow]")
|
||||
return
|
||||
|
||||
haiku = HaikuService()
|
||||
tags_service = TagAggregationService(db)
|
||||
|
||||
with Progress(
|
||||
SpinnerColumn(),
|
||||
TextColumn("[progress.description]{task.description}"),
|
||||
BarColumn(),
|
||||
TextColumn("[progress.percentage]{task.percentage:>3.0f}%"),
|
||||
console=console,
|
||||
) as progress:
|
||||
task = progress.add_task("Generating...", total=len(companies))
|
||||
|
||||
for company in companies:
|
||||
try:
|
||||
# Extract tags
|
||||
tags_response = haiku.extract_tags(
|
||||
company_name=company.name,
|
||||
website_markdown=company.website_markdown,
|
||||
address=company.address,
|
||||
)
|
||||
|
||||
# Save tags
|
||||
_save_company_tags(db, company.id, tags_response)
|
||||
|
||||
# Generate content
|
||||
content = haiku.generate_company_content(
|
||||
company_name=company.name,
|
||||
city_name=city.name,
|
||||
country_name=country.name,
|
||||
website_markdown=company.website_markdown,
|
||||
services=tags_response.services,
|
||||
industries=tags_response.industries,
|
||||
modules=tags_response.modules,
|
||||
partner_level=tags_response.partner_level,
|
||||
language=country.language,
|
||||
)
|
||||
|
||||
updates = {
|
||||
"description": content.description,
|
||||
"content_html": content.content_html,
|
||||
"meta_title": content.meta_title,
|
||||
"meta_description": content.meta_description,
|
||||
"status": "content_generated",
|
||||
"last_content_at": datetime.utcnow().isoformat(),
|
||||
}
|
||||
|
||||
db.update_company(company.id, updates)
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to generate content for {company.name}: {e}")
|
||||
|
||||
progress.advance(task)
|
||||
|
||||
# Aggregate tags for city
|
||||
console.print("Aggregating city tags...")
|
||||
tags_service.aggregate_city_tags(city.id)
|
||||
|
||||
console.print("[green]Content generation complete[/green]")
|
||||
|
||||
|
||||
def _save_company_tags(db: SupabaseService, company_id: UUID, tags_response) -> None:
|
||||
"""Save extracted tags to company."""
|
||||
# Clear existing tags
|
||||
db.remove_company_tags(company_id)
|
||||
|
||||
# Add service tags
|
||||
for name in tags_response.services:
|
||||
tag = db.get_or_create_tag(name, TagCategory.SERVICE)
|
||||
db.add_company_tag(company_id, tag.id, tags_response.confidence)
|
||||
|
||||
# Add industry tags
|
||||
for name in tags_response.industries:
|
||||
tag = db.get_or_create_tag(name, TagCategory.INDUSTRY)
|
||||
db.add_company_tag(company_id, tag.id, tags_response.confidence)
|
||||
|
||||
# Add module tags
|
||||
for name in tags_response.modules:
|
||||
tag = db.get_or_create_tag(name, TagCategory.MODULE)
|
||||
db.add_company_tag(company_id, tag.id, tags_response.confidence)
|
||||
|
||||
# Add partner level if present
|
||||
if tags_response.partner_level:
|
||||
tag = db.get_or_create_tag(tags_response.partner_level, TagCategory.PARTNER_LEVEL)
|
||||
db.add_company_tag(company_id, tag.id, tags_response.confidence)
|
||||
|
||||
|
||||
def _create_slug(name: str) -> str:
|
||||
"""Create URL-friendly slug from company name."""
|
||||
# Convert to lowercase
|
||||
slug = name.lower()
|
||||
# Replace spaces and special chars with hyphens
|
||||
slug = re.sub(r"[^a-z0-9]+", "-", slug)
|
||||
# Remove leading/trailing hyphens
|
||||
slug = slug.strip("-")
|
||||
# Limit length
|
||||
return slug[:100]
|
||||
@@ -0,0 +1,511 @@
|
||||
"""
|
||||
Export command for generating static files.
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import Optional, List, Dict, Any
|
||||
from uuid import UUID
|
||||
|
||||
import typer
|
||||
from rich.console import Console
|
||||
from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn
|
||||
from jinja2 import Environment, FileSystemLoader
|
||||
|
||||
from ..services.supabase import SupabaseService
|
||||
from ..services.tags import TagAggregationService
|
||||
from ..services.images import ImageService
|
||||
from ..config import get_settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
console = Console()
|
||||
|
||||
|
||||
def export_command(
|
||||
country: str,
|
||||
city: Optional[str] = None,
|
||||
output_dir: Optional[str] = None,
|
||||
) -> None:
|
||||
"""
|
||||
Export static XML/HTML files for a country or city.
|
||||
|
||||
Generates company pages, city hubs, country hubs, and sitemaps
|
||||
from the database content.
|
||||
|
||||
Args:
|
||||
country: Country code (e.g., DE)
|
||||
city: Optional specific city slug
|
||||
output_dir: Optional output directory override
|
||||
"""
|
||||
console.print(f"\n[bold blue]Exporting files for {country}[/bold blue]\n")
|
||||
|
||||
settings = get_settings()
|
||||
db = SupabaseService()
|
||||
tags_service = TagAggregationService(db)
|
||||
image_service = ImageService()
|
||||
|
||||
# Setup output directory
|
||||
if output_dir:
|
||||
export_path = Path(output_dir)
|
||||
else:
|
||||
export_path = settings.export_dir
|
||||
|
||||
export_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Get country info
|
||||
country_obj = db.get_country_by_code(country.upper())
|
||||
if not country_obj:
|
||||
console.print(f"[red]Country not found: {country}[/red]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
# Setup Jinja environment
|
||||
jinja_env = _setup_jinja_environment(settings.templates_dir)
|
||||
|
||||
# Get cities to export
|
||||
all_cities = db.list_cities(country_id=country_obj.id)
|
||||
|
||||
if city:
|
||||
cities_to_export = [c for c in all_cities if c.slug == city]
|
||||
if not cities_to_export:
|
||||
console.print(f"[red]City not found: {city}[/red]")
|
||||
raise typer.Exit(1)
|
||||
else:
|
||||
cities_to_export = all_cities
|
||||
|
||||
console.print(f"Exporting {len(cities_to_export)} cities...")
|
||||
|
||||
# Export each city
|
||||
total_companies = 0
|
||||
|
||||
with Progress(
|
||||
SpinnerColumn(),
|
||||
TextColumn("[progress.description]{task.description}"),
|
||||
BarColumn(),
|
||||
TextColumn("[progress.percentage]{task.percentage:>3.0f}%"),
|
||||
console=console,
|
||||
) as progress:
|
||||
task = progress.add_task("Exporting...", total=len(cities_to_export))
|
||||
|
||||
for city_obj in cities_to_export:
|
||||
city_companies = _export_city(
|
||||
db=db,
|
||||
tags_service=tags_service,
|
||||
image_service=image_service,
|
||||
jinja_env=jinja_env,
|
||||
country=country_obj,
|
||||
city=city_obj,
|
||||
export_path=export_path,
|
||||
)
|
||||
total_companies += city_companies
|
||||
progress.advance(task)
|
||||
|
||||
# Export country hub
|
||||
_export_country_hub(
|
||||
db=db,
|
||||
tags_service=tags_service,
|
||||
jinja_env=jinja_env,
|
||||
country=country_obj,
|
||||
cities=cities_to_export,
|
||||
export_path=export_path,
|
||||
)
|
||||
|
||||
# Export sitemaps
|
||||
_export_sitemaps(
|
||||
db=db,
|
||||
country=country_obj,
|
||||
cities=cities_to_export,
|
||||
export_path=export_path,
|
||||
)
|
||||
|
||||
# Export tag JSON files for frontend filtering
|
||||
_export_tag_json(
|
||||
db=db,
|
||||
tags_service=tags_service,
|
||||
country=country_obj,
|
||||
cities=cities_to_export,
|
||||
export_path=export_path,
|
||||
)
|
||||
|
||||
console.print(f"\n[bold green]Export complete![/bold green]")
|
||||
console.print(f" - Companies: {total_companies}")
|
||||
console.print(f" - Cities: {len(cities_to_export)}")
|
||||
console.print(f" - Output: {export_path}")
|
||||
|
||||
|
||||
def _setup_jinja_environment(templates_dir: Path) -> Environment:
|
||||
"""Setup Jinja2 templating environment."""
|
||||
# Create templates directory if it doesn't exist
|
||||
templates_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Create default templates if they don't exist
|
||||
_ensure_templates_exist(templates_dir)
|
||||
|
||||
return Environment(
|
||||
loader=FileSystemLoader(str(templates_dir)),
|
||||
autoescape=True,
|
||||
)
|
||||
|
||||
|
||||
def _ensure_templates_exist(templates_dir: Path) -> None:
|
||||
"""Create default templates if they don't exist."""
|
||||
# Company template
|
||||
company_template = templates_dir / "company.xml"
|
||||
if not company_template.exists():
|
||||
company_template.write_text(_DEFAULT_COMPANY_TEMPLATE)
|
||||
|
||||
# City hub template
|
||||
city_template = templates_dir / "city_hub.xml"
|
||||
if not city_template.exists():
|
||||
city_template.write_text(_DEFAULT_CITY_TEMPLATE)
|
||||
|
||||
# Country hub template
|
||||
country_template = templates_dir / "country_hub.xml"
|
||||
if not country_template.exists():
|
||||
country_template.write_text(_DEFAULT_COUNTRY_TEMPLATE)
|
||||
|
||||
|
||||
def _export_city(
|
||||
db: SupabaseService,
|
||||
tags_service: TagAggregationService,
|
||||
image_service: ImageService,
|
||||
jinja_env: Environment,
|
||||
country,
|
||||
city,
|
||||
export_path: Path,
|
||||
) -> int:
|
||||
"""Export all files for a city."""
|
||||
# Create city directory
|
||||
city_path = export_path / country.slug / city.slug
|
||||
city_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Get companies
|
||||
companies = db.list_companies(
|
||||
city_id=city.id,
|
||||
status="content_generated",
|
||||
limit=500,
|
||||
)
|
||||
|
||||
# Export each company
|
||||
company_template = jinja_env.get_template("company.xml")
|
||||
|
||||
for company in companies:
|
||||
company_tags = db.get_company_tags(company.id)
|
||||
|
||||
context = {
|
||||
"company": company,
|
||||
"city": city,
|
||||
"country": country,
|
||||
"tags": company_tags,
|
||||
"services": [t for t in company_tags if hasattr(t, "category") and t.category == "service"],
|
||||
"industries": [t for t in company_tags if hasattr(t, "category") and t.category == "industry"],
|
||||
"modules": [t for t in company_tags if hasattr(t, "category") and t.category == "module"],
|
||||
}
|
||||
|
||||
content = company_template.render(**context)
|
||||
company_file = city_path / f"{company.slug}.html"
|
||||
company_file.write_text(content, encoding="utf-8")
|
||||
|
||||
# Export city hub
|
||||
city_tags = tags_service.get_city_tags_by_category(city.id)
|
||||
hub_template = jinja_env.get_template("city_hub.xml")
|
||||
|
||||
hub_context = {
|
||||
"city": city,
|
||||
"country": country,
|
||||
"companies": companies,
|
||||
"company_count": len(companies),
|
||||
"tags": city_tags,
|
||||
}
|
||||
|
||||
hub_content = hub_template.render(**hub_context)
|
||||
hub_file = city_path / "index.html"
|
||||
hub_file.write_text(hub_content, encoding="utf-8")
|
||||
|
||||
return len(companies)
|
||||
|
||||
|
||||
def _export_country_hub(
|
||||
db: SupabaseService,
|
||||
tags_service: TagAggregationService,
|
||||
jinja_env: Environment,
|
||||
country,
|
||||
cities: List,
|
||||
export_path: Path,
|
||||
) -> None:
|
||||
"""Export country hub page."""
|
||||
country_path = export_path / country.slug
|
||||
country_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Get aggregate data
|
||||
total_companies = 0
|
||||
city_data = []
|
||||
|
||||
for city in cities:
|
||||
count = db.count_companies(city_id=city.id, status="content_generated")
|
||||
total_companies += count
|
||||
city_data.append({
|
||||
"city": city,
|
||||
"company_count": count,
|
||||
})
|
||||
|
||||
# Sort cities by company count
|
||||
city_data.sort(key=lambda x: x["company_count"], reverse=True)
|
||||
|
||||
country_tags = tags_service.get_country_tags_by_category(country.id)
|
||||
|
||||
# Render template
|
||||
template = jinja_env.get_template("country_hub.xml")
|
||||
content = template.render(
|
||||
country=country,
|
||||
cities=city_data,
|
||||
city_count=len(cities),
|
||||
company_count=total_companies,
|
||||
tags=country_tags,
|
||||
)
|
||||
|
||||
hub_file = country_path / "index.html"
|
||||
hub_file.write_text(content, encoding="utf-8")
|
||||
|
||||
|
||||
def _export_sitemaps(
|
||||
db: SupabaseService,
|
||||
country,
|
||||
cities: List,
|
||||
export_path: Path,
|
||||
) -> None:
|
||||
"""Export XML sitemaps."""
|
||||
sitemap_path = export_path / "sitemaps"
|
||||
sitemap_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
base_url = "https://odoo-expertos.com"
|
||||
|
||||
# Company sitemap
|
||||
urls = []
|
||||
for city in cities:
|
||||
companies = db.list_companies(
|
||||
city_id=city.id,
|
||||
status="content_generated",
|
||||
limit=500,
|
||||
)
|
||||
for company in companies:
|
||||
urls.append(
|
||||
f"{base_url}/{country.slug}/{city.slug}/{company.slug}.html"
|
||||
)
|
||||
|
||||
sitemap_xml = _generate_sitemap_xml(urls)
|
||||
sitemap_file = sitemap_path / f"sitemap-companies-{country.code.lower()}.xml"
|
||||
sitemap_file.write_text(sitemap_xml, encoding="utf-8")
|
||||
|
||||
# City sitemap
|
||||
city_urls = [
|
||||
f"{base_url}/{country.slug}/{city.slug}/"
|
||||
for city in cities
|
||||
]
|
||||
city_sitemap = _generate_sitemap_xml(city_urls)
|
||||
city_sitemap_file = sitemap_path / f"sitemap-cities-{country.code.lower()}.xml"
|
||||
city_sitemap_file.write_text(city_sitemap, encoding="utf-8")
|
||||
|
||||
|
||||
def _generate_sitemap_xml(urls: List[str]) -> str:
|
||||
"""Generate sitemap XML content."""
|
||||
lines = ['<?xml version="1.0" encoding="UTF-8"?>']
|
||||
lines.append('<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">')
|
||||
|
||||
for url in urls:
|
||||
lines.append(" <url>")
|
||||
lines.append(f" <loc>{url}</loc>")
|
||||
lines.append(" <changefreq>monthly</changefreq>")
|
||||
lines.append(" <priority>0.7</priority>")
|
||||
lines.append(" </url>")
|
||||
|
||||
lines.append("</urlset>")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def _export_tag_json(
|
||||
db: SupabaseService,
|
||||
tags_service: TagAggregationService,
|
||||
country,
|
||||
cities: List,
|
||||
export_path: Path,
|
||||
) -> None:
|
||||
"""Export tag aggregations as JSON for frontend filtering."""
|
||||
json_path = export_path / "tags"
|
||||
json_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# City tags
|
||||
cities_path = json_path / "cities"
|
||||
cities_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
for city in cities:
|
||||
city_json = tags_service.export_city_tags_json(city.id)
|
||||
json_file = cities_path / f"{city.slug}.json"
|
||||
json_file.write_text(json.dumps(city_json, indent=2), encoding="utf-8")
|
||||
|
||||
# Country tags
|
||||
country_json = tags_service.export_country_tags_json(country.id)
|
||||
country_file = json_path / f"{country.slug}.json"
|
||||
country_file.write_text(json.dumps(country_json, indent=2), encoding="utf-8")
|
||||
|
||||
|
||||
# Default templates
|
||||
|
||||
_DEFAULT_COMPANY_TEMPLATE = """<!DOCTYPE html>
|
||||
<html lang="{{ country.language }}">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>{{ company.meta_title or company.name }}</title>
|
||||
<meta name="description" content="{{ company.meta_description or company.description }}">
|
||||
<link rel="canonical" href="https://odoo-expertos.com/{{ country.slug }}/{{ city.slug }}/{{ company.slug }}.html">
|
||||
</head>
|
||||
<body>
|
||||
<header>
|
||||
<nav>
|
||||
<a href="/{{ country.slug }}/">{{ country.name }}</a> >
|
||||
<a href="/{{ country.slug }}/{{ city.slug }}/">{{ city.name }}</a> >
|
||||
{{ company.name }}
|
||||
</nav>
|
||||
</header>
|
||||
|
||||
<main>
|
||||
<article>
|
||||
<h1>{{ company.name }}</h1>
|
||||
|
||||
{% if company.description %}
|
||||
<p class="lead">{{ company.description }}</p>
|
||||
{% endif %}
|
||||
|
||||
<div class="company-info">
|
||||
{% if company.address %}
|
||||
<p><strong>Address:</strong> {{ company.address }}</p>
|
||||
{% endif %}
|
||||
{% if company.phone %}
|
||||
<p><strong>Phone:</strong> {{ company.phone }}</p>
|
||||
{% endif %}
|
||||
{% if company.website %}
|
||||
<p><strong>Website:</strong> <a href="{{ company.website }}" rel="noopener">{{ company.website }}</a></p>
|
||||
{% endif %}
|
||||
{% if company.email %}
|
||||
<p><strong>Email:</strong> {{ company.email }}</p>
|
||||
{% endif %}
|
||||
{% if company.rating %}
|
||||
<p><strong>Rating:</strong> {{ company.rating }}/5 ({{ company.review_count }} reviews)</p>
|
||||
{% endif %}
|
||||
</div>
|
||||
|
||||
<div class="content">
|
||||
{{ company.content_html | safe }}
|
||||
</div>
|
||||
</article>
|
||||
</main>
|
||||
|
||||
<footer>
|
||||
<p>© Odoo Expertos - {{ city.name }}, {{ country.name }}</p>
|
||||
</footer>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
|
||||
_DEFAULT_CITY_TEMPLATE = """<!DOCTYPE html>
|
||||
<html lang="{{ country.language }}">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>Odoo Partners in {{ city.name }} | {{ country.name }}</title>
|
||||
<meta name="description" content="Find {{ company_count }} certified Odoo partners in {{ city.name }}, {{ country.name }}. Compare services, expertise, and reviews.">
|
||||
<link rel="canonical" href="https://odoo-expertos.com/{{ country.slug }}/{{ city.slug }}/">
|
||||
</head>
|
||||
<body>
|
||||
<header>
|
||||
<nav>
|
||||
<a href="/{{ country.slug }}/">{{ country.name }}</a> >
|
||||
{{ city.name }}
|
||||
</nav>
|
||||
</header>
|
||||
|
||||
<main>
|
||||
<h1>Odoo Partners in {{ city.name }}</h1>
|
||||
<p>Discover {{ company_count }} certified Odoo partners in {{ city.name }}, {{ country.name }}.</p>
|
||||
|
||||
<section class="filters">
|
||||
<h2>Filter by Services</h2>
|
||||
<ul>
|
||||
{% for tag in tags.services %}
|
||||
<li>{{ tag.name }} ({{ tag.count }})</li>
|
||||
{% endfor %}
|
||||
</ul>
|
||||
</section>
|
||||
|
||||
<section class="companies">
|
||||
<h2>Odoo Partners</h2>
|
||||
{% for company in companies %}
|
||||
<article class="company-card">
|
||||
<h3><a href="{{ company.slug }}.html">{{ company.name }}</a></h3>
|
||||
{% if company.description %}
|
||||
<p>{{ company.description }}</p>
|
||||
{% endif %}
|
||||
{% if company.rating %}
|
||||
<p class="rating">{{ company.rating }}/5</p>
|
||||
{% endif %}
|
||||
</article>
|
||||
{% endfor %}
|
||||
</section>
|
||||
</main>
|
||||
|
||||
<footer>
|
||||
<p>© Odoo Expertos - {{ city.name }}, {{ country.name }}</p>
|
||||
</footer>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
|
||||
_DEFAULT_COUNTRY_TEMPLATE = """<!DOCTYPE html>
|
||||
<html lang="{{ country.language }}">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>Odoo Partners in {{ country.name }}</title>
|
||||
<meta name="description" content="Find {{ company_count }} certified Odoo partners in {{ city_count }} cities across {{ country.name }}.">
|
||||
<link rel="canonical" href="https://odoo-expertos.com/{{ country.slug }}/">
|
||||
</head>
|
||||
<body>
|
||||
<header>
|
||||
<nav>
|
||||
{{ country.name }}
|
||||
</nav>
|
||||
</header>
|
||||
|
||||
<main>
|
||||
<h1>Odoo Partners in {{ country.name }}</h1>
|
||||
<p>Discover {{ company_count }} certified Odoo partners in {{ city_count }} cities across {{ country.name }}.</p>
|
||||
|
||||
<section class="cities">
|
||||
<h2>Cities</h2>
|
||||
<ul>
|
||||
{% for item in cities %}
|
||||
<li>
|
||||
<a href="{{ item.city.slug }}/">{{ item.city.name }}</a>
|
||||
({{ item.company_count }} partners)
|
||||
</li>
|
||||
{% endfor %}
|
||||
</ul>
|
||||
</section>
|
||||
|
||||
<section class="filters">
|
||||
<h2>Services</h2>
|
||||
<ul>
|
||||
{% for tag in tags.services %}
|
||||
<li>{{ tag.name }} ({{ tag.company_count }} companies in {{ tag.city_count }} cities)</li>
|
||||
{% endfor %}
|
||||
</ul>
|
||||
</section>
|
||||
</main>
|
||||
|
||||
<footer>
|
||||
<p>© Odoo Expertos - {{ country.name }}</p>
|
||||
</footer>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
@@ -0,0 +1,242 @@
|
||||
"""
|
||||
Plan command for creating execution plans.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from typing import List, Optional
|
||||
from uuid import UUID
|
||||
|
||||
import typer
|
||||
from rich.console import Console
|
||||
from rich.table import Table
|
||||
|
||||
from ..services.supabase import SupabaseService
|
||||
from ..models.schemas import ExecutionPlan, PlanStep, StepStatus
|
||||
from ..config import SEARCH_QUERIES
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
console = Console()
|
||||
|
||||
|
||||
def plan_command(
|
||||
country: str,
|
||||
cities: Optional[List[str]] = None,
|
||||
name: Optional[str] = None,
|
||||
) -> Optional[UUID]:
|
||||
"""
|
||||
Create an execution plan for scraping Odoo partners.
|
||||
|
||||
This generates a plan that specifies which cities to scrape and
|
||||
what steps to perform. The plan can then be executed with the
|
||||
execute command.
|
||||
|
||||
Args:
|
||||
country: Country code (e.g., DE for Germany)
|
||||
cities: List of city slugs to include (default: all cities in country)
|
||||
name: Optional name for the plan
|
||||
|
||||
Returns:
|
||||
Plan UUID if successful
|
||||
"""
|
||||
console.print(f"\n[bold blue]Creating Execution Plan for {country}[/bold blue]\n")
|
||||
|
||||
db = SupabaseService()
|
||||
|
||||
# Verify country exists
|
||||
country_obj = db.get_country_by_code(country.upper())
|
||||
if not country_obj:
|
||||
console.print(f"[red]Country not found: {country}[/red]")
|
||||
console.print("Run 'setup' command first to seed country data.")
|
||||
raise typer.Exit(1)
|
||||
|
||||
# Get cities
|
||||
all_cities = db.list_cities(country_id=country_obj.id)
|
||||
|
||||
if cities:
|
||||
# Filter to specified cities
|
||||
city_slugs = [c.lower() for c in cities]
|
||||
selected_cities = [c for c in all_cities if c.slug in city_slugs]
|
||||
|
||||
missing = set(city_slugs) - {c.slug for c in selected_cities}
|
||||
if missing:
|
||||
console.print(f"[yellow]Warning: Cities not found: {missing}[/yellow]")
|
||||
else:
|
||||
selected_cities = all_cities
|
||||
|
||||
if not selected_cities:
|
||||
console.print("[red]No cities found for this country.[/red]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
# Generate plan name
|
||||
if not name:
|
||||
city_names = [c.slug for c in selected_cities[:3]]
|
||||
name = f"{country.upper()}-{'-'.join(city_names)}"
|
||||
if len(selected_cities) > 3:
|
||||
name += f"-and-{len(selected_cities) - 3}-more"
|
||||
|
||||
# Estimate company counts based on queries
|
||||
language = country_obj.language
|
||||
queries_per_city = len(SEARCH_QUERIES.get(language, SEARCH_QUERIES["en"]))
|
||||
estimated_companies = len(selected_cities) * queries_per_city * 15 # ~15 per query
|
||||
|
||||
# Show plan summary
|
||||
table = Table(title="Execution Plan Summary")
|
||||
table.add_column("Field", style="cyan")
|
||||
table.add_column("Value", style="green")
|
||||
|
||||
table.add_row("Plan Name", name)
|
||||
table.add_row("Country", f"{country_obj.name} ({country_obj.code})")
|
||||
table.add_row("Language", country_obj.language)
|
||||
table.add_row("Cities", str(len(selected_cities)))
|
||||
table.add_row("Queries/City", str(queries_per_city))
|
||||
table.add_row("Est. Companies", f"~{estimated_companies}")
|
||||
|
||||
console.print(table)
|
||||
console.print()
|
||||
|
||||
# Show city list
|
||||
console.print("[bold]Cities included:[/bold]")
|
||||
for city in selected_cities:
|
||||
console.print(f" - {city.name} ({city.slug})")
|
||||
|
||||
console.print()
|
||||
|
||||
# Confirm creation
|
||||
if not typer.confirm("Create this plan?"):
|
||||
console.print("[yellow]Plan creation cancelled.[/yellow]")
|
||||
raise typer.Exit(0)
|
||||
|
||||
# Create the plan
|
||||
plan = ExecutionPlan(
|
||||
name=name,
|
||||
country_code=country.upper(),
|
||||
cities=[c.slug for c in selected_cities],
|
||||
status=StepStatus.PENDING,
|
||||
)
|
||||
|
||||
plan = db.create_execution_plan(plan)
|
||||
console.print(f"\n[green]Plan created: {plan.id}[/green]")
|
||||
|
||||
# Create steps for each city
|
||||
step_types = ["scrape", "enrich", "content", "export"]
|
||||
|
||||
for city in selected_cities:
|
||||
for step_type in step_types:
|
||||
step = PlanStep(
|
||||
plan_id=plan.id,
|
||||
step_name=step_type,
|
||||
city_id=city.id,
|
||||
status=StepStatus.PENDING,
|
||||
)
|
||||
db.create_plan_step(step)
|
||||
|
||||
console.print(
|
||||
f"[green]Created {len(selected_cities) * len(step_types)} steps[/green]"
|
||||
)
|
||||
|
||||
# Print next steps
|
||||
console.print("\n[bold]Next steps:[/bold]")
|
||||
console.print(f" 1. Run: python -m directory.cli execute --plan {plan.id}")
|
||||
console.print(" 2. Or run step by step:")
|
||||
console.print(f" python -m directory.cli execute --plan {plan.id} --step scrape")
|
||||
console.print(
|
||||
f" python -m directory.cli execute --plan {plan.id} --step enrich"
|
||||
)
|
||||
console.print(
|
||||
f" python -m directory.cli execute --plan {plan.id} --step content"
|
||||
)
|
||||
console.print()
|
||||
|
||||
return plan.id
|
||||
|
||||
|
||||
def list_plans() -> None:
|
||||
"""List all execution plans."""
|
||||
db = SupabaseService()
|
||||
plans = db.list_execution_plans()
|
||||
|
||||
if not plans:
|
||||
console.print("[yellow]No execution plans found.[/yellow]")
|
||||
return
|
||||
|
||||
table = Table(title="Execution Plans")
|
||||
table.add_column("ID", style="cyan", max_width=36)
|
||||
table.add_column("Name", style="green")
|
||||
table.add_column("Country", style="blue")
|
||||
table.add_column("Cities", style="magenta")
|
||||
table.add_column("Status", style="yellow")
|
||||
table.add_column("Created", style="dim")
|
||||
|
||||
for plan in plans:
|
||||
table.add_row(
|
||||
str(plan.id),
|
||||
plan.name,
|
||||
plan.country_code,
|
||||
str(len(plan.cities)),
|
||||
plan.status.value if hasattr(plan.status, "value") else plan.status,
|
||||
str(plan.created_at)[:16] if plan.created_at else "-",
|
||||
)
|
||||
|
||||
console.print(table)
|
||||
|
||||
|
||||
def show_plan(plan_id: str) -> None:
|
||||
"""Show details of a specific plan."""
|
||||
db = SupabaseService()
|
||||
|
||||
try:
|
||||
plan_uuid = UUID(plan_id)
|
||||
except ValueError:
|
||||
console.print(f"[red]Invalid plan ID: {plan_id}[/red]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
plan = db.get_execution_plan(plan_uuid)
|
||||
|
||||
if not plan:
|
||||
console.print(f"[red]Plan not found: {plan_id}[/red]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
# Plan info
|
||||
console.print(f"\n[bold blue]Plan: {plan.name}[/bold blue]\n")
|
||||
|
||||
table = Table()
|
||||
table.add_column("Field", style="cyan")
|
||||
table.add_column("Value", style="green")
|
||||
|
||||
table.add_row("ID", str(plan.id))
|
||||
table.add_row("Country", plan.country_code)
|
||||
table.add_row("Cities", ", ".join(plan.cities))
|
||||
table.add_row("Status", plan.status.value if hasattr(plan.status, "value") else plan.status)
|
||||
table.add_row("Created", str(plan.created_at)[:19] if plan.created_at else "-")
|
||||
table.add_row("Started", str(plan.started_at)[:19] if plan.started_at else "-")
|
||||
table.add_row("Completed", str(plan.completed_at)[:19] if plan.completed_at else "-")
|
||||
|
||||
console.print(table)
|
||||
|
||||
# Steps
|
||||
if plan.steps:
|
||||
console.print("\n[bold]Steps:[/bold]")
|
||||
|
||||
steps_table = Table()
|
||||
steps_table.add_column("Step", style="cyan")
|
||||
steps_table.add_column("City", style="blue")
|
||||
steps_table.add_column("Status", style="yellow")
|
||||
steps_table.add_column("Progress", style="green")
|
||||
|
||||
for step in plan.steps:
|
||||
city_slug = "-"
|
||||
if step.city_id:
|
||||
city = db.get_city_by_slug("", step.city_id)
|
||||
# We don't have a get_city_by_id, so we'll just show the ID
|
||||
city_slug = str(step.city_id)[:8]
|
||||
|
||||
progress = f"{step.items_processed}/{step.items_total}" if step.items_total else "-"
|
||||
|
||||
steps_table.add_row(
|
||||
step.step_name,
|
||||
city_slug,
|
||||
step.status.value if hasattr(step.status, "value") else step.status,
|
||||
progress,
|
||||
)
|
||||
|
||||
console.print(steps_table)
|
||||
@@ -0,0 +1,304 @@
|
||||
"""
|
||||
Setup command for initializing the database.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from typing import Optional
|
||||
|
||||
import typer
|
||||
from rich.console import Console
|
||||
from rich.progress import Progress, SpinnerColumn, TextColumn
|
||||
|
||||
from ..services.supabase import SupabaseService
|
||||
from ..models.schemas import Region, Country, City, Tag, TagCategory
|
||||
from ..config import DEFAULT_TAGS, GERMANY_CITIES
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
console = Console()
|
||||
|
||||
|
||||
# SQL schema for creating tables
|
||||
SCHEMA_SQL = """
|
||||
-- Enable UUID extension
|
||||
CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
|
||||
|
||||
-- Regions table
|
||||
CREATE TABLE IF NOT EXISTS regions (
|
||||
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||
name TEXT NOT NULL,
|
||||
slug TEXT NOT NULL UNIQUE,
|
||||
created_at TIMESTAMPTZ DEFAULT NOW()
|
||||
);
|
||||
|
||||
-- Countries table
|
||||
CREATE TABLE IF NOT EXISTS countries (
|
||||
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||
region_id UUID REFERENCES regions(id),
|
||||
name TEXT NOT NULL,
|
||||
name_local TEXT,
|
||||
code TEXT NOT NULL UNIQUE,
|
||||
slug TEXT NOT NULL UNIQUE,
|
||||
language TEXT NOT NULL,
|
||||
created_at TIMESTAMPTZ DEFAULT NOW()
|
||||
);
|
||||
|
||||
-- Cities table
|
||||
CREATE TABLE IF NOT EXISTS cities (
|
||||
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||
country_id UUID REFERENCES countries(id),
|
||||
name TEXT NOT NULL,
|
||||
name_local TEXT,
|
||||
slug TEXT NOT NULL,
|
||||
state TEXT,
|
||||
population INTEGER,
|
||||
latitude FLOAT,
|
||||
longitude FLOAT,
|
||||
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
UNIQUE(country_id, slug)
|
||||
);
|
||||
|
||||
-- Tags table
|
||||
CREATE TABLE IF NOT EXISTS tags (
|
||||
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||
category TEXT NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
slug TEXT NOT NULL UNIQUE,
|
||||
description TEXT,
|
||||
created_at TIMESTAMPTZ DEFAULT NOW()
|
||||
);
|
||||
|
||||
-- Companies table
|
||||
CREATE TABLE IF NOT EXISTS companies (
|
||||
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||
city_id UUID REFERENCES cities(id),
|
||||
name TEXT NOT NULL,
|
||||
slug TEXT NOT NULL,
|
||||
google_place_id TEXT UNIQUE,
|
||||
address TEXT,
|
||||
phone TEXT,
|
||||
website TEXT,
|
||||
email TEXT,
|
||||
rating FLOAT,
|
||||
review_count INTEGER,
|
||||
latitude FLOAT,
|
||||
longitude FLOAT,
|
||||
website_markdown TEXT,
|
||||
logo_url TEXT,
|
||||
description TEXT,
|
||||
content_html TEXT,
|
||||
meta_title TEXT,
|
||||
meta_description TEXT,
|
||||
status TEXT DEFAULT 'scraped',
|
||||
last_scraped_at TIMESTAMPTZ,
|
||||
last_enriched_at TIMESTAMPTZ,
|
||||
last_content_at TIMESTAMPTZ,
|
||||
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ DEFAULT NOW()
|
||||
);
|
||||
|
||||
-- Company tags junction table
|
||||
CREATE TABLE IF NOT EXISTS company_tags (
|
||||
company_id UUID REFERENCES companies(id) ON DELETE CASCADE,
|
||||
tag_id UUID REFERENCES tags(id) ON DELETE CASCADE,
|
||||
confidence FLOAT DEFAULT 1.0,
|
||||
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
PRIMARY KEY (company_id, tag_id)
|
||||
);
|
||||
|
||||
-- City tag aggregation
|
||||
CREATE TABLE IF NOT EXISTS city_tags (
|
||||
city_id UUID REFERENCES cities(id) ON DELETE CASCADE,
|
||||
tag_id UUID REFERENCES tags(id) ON DELETE CASCADE,
|
||||
company_count INTEGER DEFAULT 0,
|
||||
PRIMARY KEY (city_id, tag_id)
|
||||
);
|
||||
|
||||
-- Country tag aggregation
|
||||
CREATE TABLE IF NOT EXISTS country_tags (
|
||||
country_id UUID REFERENCES countries(id) ON DELETE CASCADE,
|
||||
tag_id UUID REFERENCES tags(id) ON DELETE CASCADE,
|
||||
company_count INTEGER DEFAULT 0,
|
||||
city_count INTEGER DEFAULT 0,
|
||||
PRIMARY KEY (country_id, tag_id)
|
||||
);
|
||||
|
||||
-- Execution plans
|
||||
CREATE TABLE IF NOT EXISTS execution_plans (
|
||||
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||
name TEXT NOT NULL,
|
||||
country_code TEXT NOT NULL,
|
||||
cities TEXT[] NOT NULL,
|
||||
status TEXT DEFAULT 'pending',
|
||||
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
started_at TIMESTAMPTZ,
|
||||
completed_at TIMESTAMPTZ
|
||||
);
|
||||
|
||||
-- Plan steps
|
||||
CREATE TABLE IF NOT EXISTS plan_steps (
|
||||
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||
plan_id UUID REFERENCES execution_plans(id) ON DELETE CASCADE,
|
||||
step_name TEXT NOT NULL,
|
||||
city_id UUID REFERENCES cities(id),
|
||||
status TEXT DEFAULT 'pending',
|
||||
started_at TIMESTAMPTZ,
|
||||
completed_at TIMESTAMPTZ,
|
||||
error_message TEXT,
|
||||
items_processed INTEGER DEFAULT 0,
|
||||
items_total INTEGER DEFAULT 0
|
||||
);
|
||||
|
||||
-- Error logs
|
||||
CREATE TABLE IF NOT EXISTS error_logs (
|
||||
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||
entity_type TEXT NOT NULL,
|
||||
entity_id UUID,
|
||||
operation TEXT NOT NULL,
|
||||
error_message TEXT NOT NULL,
|
||||
details JSONB DEFAULT '{}',
|
||||
created_at TIMESTAMPTZ DEFAULT NOW()
|
||||
);
|
||||
|
||||
-- Indexes for performance
|
||||
CREATE INDEX IF NOT EXISTS idx_companies_city ON companies(city_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_companies_status ON companies(status);
|
||||
CREATE INDEX IF NOT EXISTS idx_company_tags_company ON company_tags(company_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_company_tags_tag ON company_tags(tag_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_city_tags_city ON city_tags(city_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_country_tags_country ON country_tags(country_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_plan_steps_plan ON plan_steps(plan_id);
|
||||
"""
|
||||
|
||||
|
||||
def setup_command(
|
||||
seed_tags: bool = True,
|
||||
seed_germany: bool = True,
|
||||
) -> None:
|
||||
"""
|
||||
Initialize the database with schema and seed data.
|
||||
|
||||
This command should be run once when setting up a new installation.
|
||||
It creates all required tables and optionally seeds reference data.
|
||||
|
||||
Args:
|
||||
seed_tags: Whether to seed default tags
|
||||
seed_germany: Whether to seed Germany cities
|
||||
"""
|
||||
console.print("\n[bold blue]Odoo Directory Setup[/bold blue]\n")
|
||||
|
||||
db = SupabaseService()
|
||||
|
||||
with Progress(
|
||||
SpinnerColumn(),
|
||||
TextColumn("[progress.description]{task.description}"),
|
||||
console=console,
|
||||
) as progress:
|
||||
# Note: Schema should be created in Supabase SQL Editor
|
||||
# The CLI just verifies connectivity and seeds data
|
||||
|
||||
task = progress.add_task("Verifying database connection...", total=None)
|
||||
try:
|
||||
# Test connection by listing regions
|
||||
regions = db.list_regions()
|
||||
progress.update(task, description="[green]Database connected")
|
||||
except Exception as e:
|
||||
console.print(f"\n[red]Database connection failed: {e}[/red]")
|
||||
console.print(
|
||||
"\n[yellow]Please create tables using the SQL schema in Supabase SQL Editor.[/yellow]"
|
||||
)
|
||||
console.print("\nSchema SQL can be found in directory/commands/setup.py")
|
||||
raise typer.Exit(1)
|
||||
|
||||
# Seed tags
|
||||
if seed_tags:
|
||||
task = progress.add_task("Seeding tags...", total=None)
|
||||
tag_count = _seed_tags(db)
|
||||
progress.update(task, description=f"[green]Seeded {tag_count} tags")
|
||||
|
||||
# Seed Germany
|
||||
if seed_germany:
|
||||
task = progress.add_task("Seeding Germany data...", total=None)
|
||||
city_count = _seed_germany(db)
|
||||
progress.update(task, description=f"[green]Seeded {city_count} cities")
|
||||
|
||||
console.print("\n[bold green]Setup complete![/bold green]\n")
|
||||
|
||||
|
||||
def _seed_tags(db: SupabaseService) -> int:
|
||||
"""Seed default tags into database."""
|
||||
count = 0
|
||||
|
||||
# Map category names to enum values
|
||||
category_map = {
|
||||
"services": TagCategory.SERVICE,
|
||||
"industries": TagCategory.INDUSTRY,
|
||||
"modules": TagCategory.MODULE,
|
||||
"partner_levels": TagCategory.PARTNER_LEVEL,
|
||||
}
|
||||
|
||||
for category_name, tag_names in DEFAULT_TAGS.items():
|
||||
category = category_map[category_name]
|
||||
|
||||
for name in tag_names:
|
||||
slug = name.lower().replace(" ", "-").replace("&", "and")
|
||||
existing = db.get_tag_by_slug(slug)
|
||||
|
||||
if not existing:
|
||||
db.create_tag(
|
||||
Tag(
|
||||
category=category,
|
||||
name=name,
|
||||
slug=slug,
|
||||
)
|
||||
)
|
||||
count += 1
|
||||
|
||||
return count
|
||||
|
||||
|
||||
def _seed_germany(db: SupabaseService) -> int:
|
||||
"""Seed Germany region, country, and cities."""
|
||||
# Create Europe region if not exists
|
||||
region = db.get_region_by_slug("europe")
|
||||
if not region:
|
||||
region = db.create_region(Region(name="Europe", slug="europe"))
|
||||
|
||||
# Create Germany if not exists
|
||||
country = db.get_country_by_code("DE")
|
||||
if not country:
|
||||
country = db.create_country(
|
||||
Country(
|
||||
region_id=region.id,
|
||||
name="Germany",
|
||||
name_local="Deutschland",
|
||||
code="DE",
|
||||
slug="deutschland",
|
||||
language="de",
|
||||
)
|
||||
)
|
||||
|
||||
# Create cities
|
||||
count = 0
|
||||
for city_data in GERMANY_CITIES:
|
||||
existing = db.get_city_by_slug(city_data["slug"], country.id)
|
||||
if not existing:
|
||||
db.create_city(
|
||||
City(
|
||||
country_id=country.id,
|
||||
name=city_data["name"],
|
||||
slug=city_data["slug"],
|
||||
state=city_data.get("state"),
|
||||
)
|
||||
)
|
||||
count += 1
|
||||
|
||||
return count
|
||||
|
||||
|
||||
def print_schema() -> None:
|
||||
"""Print the SQL schema for manual creation."""
|
||||
console.print("\n[bold]Database Schema SQL:[/bold]\n")
|
||||
console.print(SCHEMA_SQL)
|
||||
console.print(
|
||||
"\n[yellow]Copy this SQL and run it in your Supabase SQL Editor.[/yellow]"
|
||||
)
|
||||
Reference in New Issue
Block a user