new site

directory/services/apify.py

@@ -0,0 +1,224 @@
+"""
+Apify service for Google Maps scraping.
+"""
+
+import logging
+from typing import List, Optional, Dict, Any
+import time
+
+import httpx
+
+from ..config import get_settings, APIFY_GOOGLE_MAPS_ACTOR, SEARCH_QUERIES
+from ..models.schemas import ApifyGoogleMapsResult
+from ..utils.retry import retry_with_backoff
+
+logger = logging.getLogger(__name__)
+
+
+class ApifyService:
+    """Service for interacting with Apify Google Maps scraper."""
+
+    BASE_URL = "https://api.apify.com/v2"
+
+    def __init__(self):
+        """Initialize Apify client."""
+        settings = get_settings()
+        self.token = settings.apify_token
+        self.headers = {"Authorization": f"Bearer {self.token}"}
+
+    def _get_search_queries(self, city: str, language: str = "de") -> List[str]:
+        """
+        Generate search queries for a city.
+
+        Args:
+            city: City name
+            language: Language code for queries
+
+        Returns:
+            List of search query strings
+        """
+        templates = SEARCH_QUERIES.get(language, SEARCH_QUERIES["en"])
+        return [template.format(city=city) for template in templates]
+
+    @retry_with_backoff(max_attempts=3, exceptions=(httpx.HTTPError,))
+    def run_google_maps_scraper(
+        self,
+        search_queries: List[str],
+        max_results_per_query: int = 20,
+        language: str = "de",
+        country: str = "DE",
+    ) -> str:
+        """
+        Start a Google Maps scraper run.
+
+        Args:
+            search_queries: List of search queries
+            max_results_per_query: Max results per query
+            language: Language for results
+            country: Country code for search
+
+        Returns:
+            Run ID for tracking
+        """
+        url = f"{self.BASE_URL}/acts/{APIFY_GOOGLE_MAPS_ACTOR}/runs"
+
+        payload = {
+            "searchStringsArray": search_queries,
+            "maxCrawledPlaces": max_results_per_query * len(search_queries),
+            "language": language,
+            "countryCode": country,
+            "maxReviews": 0,  # Don't fetch reviews to save credits
+            "maxImages": 0,  # Don't fetch images
+            "includeWebResults": False,
+        }
+
+        with httpx.Client(timeout=60.0) as client:
+            response = client.post(url, json=payload, headers=self.headers)
+            response.raise_for_status()
+            data = response.json()
+            run_id = data["data"]["id"]
+            logger.info(f"Started Apify run: {run_id}")
+            return run_id
+
+    def wait_for_run(
+        self, run_id: str, timeout: int = 300, poll_interval: int = 10
+    ) -> str:
+        """
+        Wait for an Apify run to complete.
+
+        Args:
+            run_id: Run ID to wait for
+            timeout: Maximum wait time in seconds
+            poll_interval: Time between status checks
+
+        Returns:
+            Final status of the run
+        """
+        url = f"{self.BASE_URL}/actor-runs/{run_id}"
+        start_time = time.time()
+
+        while True:
+            if time.time() - start_time > timeout:
+                raise TimeoutError(f"Apify run {run_id} timed out after {timeout}s")
+
+            with httpx.Client(timeout=30.0) as client:
+                response = client.get(url, headers=self.headers)
+                response.raise_for_status()
+                data = response.json()
+
+            status = data["data"]["status"]
+            logger.debug(f"Run {run_id} status: {status}")
+
+            if status in ("SUCCEEDED", "FAILED", "ABORTED", "TIMED-OUT"):
+                return status
+
+            time.sleep(poll_interval)
+
+    def get_run_results(self, run_id: str) -> List[ApifyGoogleMapsResult]:
+        """
+        Get results from a completed Apify run.
+
+        Args:
+            run_id: Run ID to get results for
+
+        Returns:
+            List of Google Maps results
+        """
+        url = f"{self.BASE_URL}/actor-runs/{run_id}/dataset/items"
+
+        with httpx.Client(timeout=60.0) as client:
+            response = client.get(url, headers=self.headers)
+            response.raise_for_status()
+            data = response.json()
+
+        results = []
+        for item in data:
+            try:
+                result = ApifyGoogleMapsResult(
+                    title=item.get("title", ""),
+                    address=item.get("address"),
+                    phone=item.get("phone"),
+                    website=item.get("website"),
+                    rating=item.get("totalScore"),
+                    reviewsCount=item.get("reviewsCount"),
+                    placeId=item.get("placeId"),
+                    url=item.get("url"),
+                    location=item.get("location"),
+                )
+                results.append(result)
+            except Exception as e:
+                logger.warning(f"Failed to parse result: {e}")
+
+        logger.info(f"Retrieved {len(results)} results from run {run_id}")
+        return results
+
+    def scrape_city(
+        self,
+        city_name: str,
+        language: str = "de",
+        country: str = "DE",
+        max_per_query: int = 20,
+    ) -> List[ApifyGoogleMapsResult]:
+        """
+        Scrape Google Maps for Odoo partners in a city.
+
+        This is the main high-level method that combines query generation,
+        running the scraper, and retrieving results.
+
+        Args:
+            city_name: Name of the city to scrape
+            language: Language code
+            country: Country code
+            max_per_query: Max results per search query
+
+        Returns:
+            List of Google Maps results
+        """
+        queries = self._get_search_queries(city_name, language)
+        logger.info(f"Scraping {city_name} with {len(queries)} queries")
+
+        # Start the run
+        run_id = self.run_google_maps_scraper(
+            search_queries=queries,
+            max_results_per_query=max_per_query,
+            language=language,
+            country=country,
+        )
+
+        # Wait for completion
+        status = self.wait_for_run(run_id)
+
+        if status != "SUCCEEDED":
+            raise RuntimeError(f"Apify run failed with status: {status}")
+
+        # Get results
+        return self.get_run_results(run_id)
+
+    def deduplicate_results(
+        self, results: List[ApifyGoogleMapsResult]
+    ) -> List[ApifyGoogleMapsResult]:
+        """
+        Remove duplicate results based on place ID.
+
+        Args:
+            results: List of results to deduplicate
+
+        Returns:
+            Deduplicated list
+        """
+        seen_place_ids = set()
+        unique_results = []
+
+        for result in results:
+            if result.placeId and result.placeId not in seen_place_ids:
+                seen_place_ids.add(result.placeId)
+                unique_results.append(result)
+            elif not result.placeId:
+                # Keep results without place ID but log warning
+                logger.warning(f"Result without place ID: {result.title}")
+                unique_results.append(result)
+
+        logger.info(
+            f"Deduplicated {len(results)} results to {len(unique_results)} unique"
+        )
+        return unique_results