API Docs

import requests
import json
import time
from datetime import datetime

def scrape_facebook_async(api_key, facebook_url, num_posts,
                          output_dir="./scraped_data",
                          poll_interval=3,
                          timeout=300):
    """
    Scrape Facebook posts using async job-based API

    Args:
        api_key: Your API key
        facebook_url: Facebook profile/page URL
        num_posts: Number of posts to scrape
        output_dir: Directory to save results
        poll_interval: Seconds between status checks
        timeout: Max seconds to wait for completion

    Returns:
        dict with success status and data
    """
    import os
    os.makedirs(output_dir, exist_ok=True)

    base_url = "https://scraper-api-e5h6.onrender.com/api/v1"
    headers = {
        "X-API-Key": api_key,
        "Content-Type": "application/json"
    }

    print(f"Creating scraping job for {facebook_url}...")
    print(f"Requesting {num_posts} posts")

    # Step 1: Create the job
    try:
        response = requests.post(
            f"{base_url}/scraping/api/scrape",
            headers=headers,
            json={
                "facebook_url": facebook_url,
                "num_posts": num_posts
            }
        )

        if response.status_code != 200:
            error = response.json().get('detail', 'Unknown error')
            print(f"Error creating job: {error}")
            return {"success": False, "error": error}

        job_data = response.json()
        job_id = job_data["job_id"]
        status_url = f"{base_url}{job_data['status_check_url']}"

        print(f"\n✓ Job created successfully")
        print(f"  Job ID: {job_id}")
        print(f"  Status: {job_data['status']}")
        print(f"  Estimated time: {job_data['estimated_time_seconds']}s")

    except Exception as e:
        print(f"Error: {e}")
        return {"success": False, "error": str(e)}

    # Step 2: Poll for completion
    print(f"\nMonitoring job progress...")
    start_time = time.time()

    while True:
        elapsed = time.time() - start_time

        # Check timeout
        if elapsed > timeout:
            print(f"\n⚠ Timeout after {timeout}s")
            return {"success": False, "error": "Timeout"}

        # Check status
        try:
            response = requests.get(status_url, headers=headers)
            status_data = response.json()

            status = status_data["status"]
            posts_scraped = status_data.get("posts_scraped", 0)

            # Update progress
            if status == "processing":
                print(f"  [{elapsed:.0f}s] Processing... {posts_scraped} posts scraped")
            elif status == "pending":
                print(f"  [{elapsed:.0f}s] Waiting in queue...")
            elif status == "completed":
                # Success! Save the data
                print(f"\n✓ Scraping completed!")
                print(f"  Posts scraped: {posts_scraped}")
                print(f"  Duration: {elapsed:.1f}s")

                filename = f"facebook_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json"
                filepath = os.path.join(output_dir, filename)

                with open(filepath, "w", encoding="utf-8") as f:
                    json.dump(status_data["result"], f, indent=2, ensure_ascii=False)

                print(f"  File saved: {filepath}")

                return {
                    "success": True,
                    "filepath": filepath,
                    "posts_scraped": posts_scraped,
                    "duration": elapsed
                }
            elif status == "failed":
                error = status_data.get("error", "Unknown error")
                print(f"\n✗ Scraping failed: {error}")
                return {"success": False, "error": error}

            # Wait before next check
            time.sleep(poll_interval)

        except Exception as e:
            print(f"\nError checking status: {e}")
            return {"success": False, "error": str(e)}

# Usage example
if __name__ == "__main__":
    result = scrape_facebook_async(
        api_key="sk_your_api_key_here",
        facebook_url="https://www.facebook.com/zuck",
        num_posts=10,
        poll_interval=3,
        timeout=300
    )

    if result["success"]:
        print(f"\nSuccess! Data saved to {result['filepath']}")
    else:
        print(f"\nFailed: {result['error']}")