Monitor Brand Impersonation

import json
import os
import random
import time
from pathlib import Path

import requests

BASE_URL = "https://fullhunt.io/api/v1/enterprise/darkweb"
API_KEY = os.environ["FULLHUNT_API_KEY"]
MONITORED_DOMAIN = os.environ["FULLHUNT_MONITORED_DOMAIN"]
STATE_PATH = Path(os.environ.get("FULLHUNT_STATE_PATH", "brand-monitor-state.json"))
HEADERS = {"X-API-KEY": API_KEY}
ENDPOINTS = ("potential-phishing", "typosquatting")


def request_page(session, endpoint, page, max_attempts=5):
    params = {"q": MONITORED_DOMAIN, "page": page}

    for attempt in range(max_attempts):
        response = session.get(
            f"{BASE_URL}/{endpoint}",
            headers=HEADERS,
            params=params,
            timeout=30,
        )
        if response.status_code == 429 or response.status_code >= 500:
            retry_after = response.headers.get("Retry-After")
            delay = float(retry_after) if retry_after else min(2**attempt, 30)
            time.sleep(delay + random.uniform(0, 1))
            continue

        response.raise_for_status()
        payload = response.json()
        results = payload.get("results")
        if not isinstance(results, dict) or not isinstance(results.get("items"), list):
            raise ValueError(f"Unexpected response shape from {endpoint}")
        return results

    raise RuntimeError(f"{endpoint} did not recover after {max_attempts} attempts")


def fetch_all(session, endpoint):
    records = []
    page = 1

    while True:
        results = request_page(session, endpoint, page)
        records.extend(results["items"])
        pages = int(results.get("pages") or 0)
        if not results["items"] or page >= pages:
            return records
        page += 1


def load_seen():
    if not STATE_PATH.exists():
        return set()
    data = json.loads(STATE_PATH.read_text(encoding="utf-8"))
    return set(data.get("seen", []))


def save_seen(seen):
    temporary = STATE_PATH.with_suffix(STATE_PATH.suffix + ".tmp")
    temporary.write_text(
        json.dumps({"seen": sorted(seen)}, indent=2) + "\n",
        encoding="utf-8",
    )
    temporary.replace(STATE_PATH)


def record_key(endpoint, record):
    stable_id = record.get("id") or record.get("domain_name")
    if not stable_id:
        raise ValueError(f"Record from {endpoint} has no stable identifier")
    return f"{endpoint}:{stable_id}"


def main():
    seen = load_seen()
    updated_seen = set(seen)
    new_records = []

    with requests.Session() as session:
        for endpoint in ENDPOINTS:
            for record in fetch_all(session, endpoint):
                key = record_key(endpoint, record)
                updated_seen.add(key)
                if key not in seen:
                    new_records.append(
                        {
                            "dataset": endpoint,
                            "domain": record.get("domain_name"),
                            "first_seen": record.get("date_added"),
                            "last_seen": record.get("last_seen"),
                            "record_id": record.get("id"),
                        }
                    )

    save_seen(updated_seen)
    print(json.dumps({"new_candidates": new_records}, indent=2))


if __name__ == "__main__":
    main()