Bulk SERP API for large-scale Google data collection

import requests

API = "https://api.semscraper.com/v1/serp"
HEADERS = {"Authorization": "Bearer API_KEY"}

keywords = open("keywords.txt").read().splitlines()  # e.g. 10,000 keywords

# 1,000 keywords per call, without waiting for the previous batch
for i in range(0, len(keywords), 1000):
    batch = [{
        "search_engine": "google_search",
        "keyword": kw,
        "device": "desktop",
        "location": "en",
        "language": "en",
        "depth": 1,
        "callback_url": "https://your-server.com/serp-callback",
    } for kw in keywords[i:i + 1000]]
    r = requests.post(API, json=batch, headers=HEADERS, timeout=60)
    r.raise_for_status()
    print(r.json()["count"], "keywords queued")

import hashlib, hmac, json
from flask import Flask, request

SECRET = b"hmac_..."  # HMAC key from your API keys menu
app = Flask(__name__)

@app.post("/serp-callback")
def serp_callback():
    raw = request.get_data()
    expected = hmac.new(SECRET, raw, hashlib.sha256).hexdigest()
    if not hmac.compare_digest(expected, request.headers.get("X-Signature", "")):
        return "Invalid signature", 401
    for serp in json.loads(raw):  # a list holding the finished SERP
        save(serp)                # your own storage
    return {"status": "ok"}       # anything but 200 leaves it unfetched

# finished results that were never fetched, up to 1,000 per call
while True:
    pending = requests.get(API, headers=HEADERS, timeout=60).json().get("data", [])
    if not pending:
        break
    ids = [p["id"] for p in pending]
    for i in range(0, len(ids), 100):
        r = requests.get(API, headers=HEADERS, timeout=60,
                         params={"ids": ",".join(ids[i:i + 100]), "output": "json"})
        for serp in r.json()["data"]:
            save(serp)  # fetching marks it as fetched, and is never billed