SERPの類似度によるキーワードクラスタリング

import time
import requests

API_KEY = "API_KEY"
API = "https://api.semscraper.com/v1/serp"
HEADERS = {"Authorization": "Bearer " + API_KEY}
THRESHOLD = 3

# 検索回数の多い順
keywords = ["ランニングシューズ レディース", "女性用 ジョギング スニーカー", "トレイルランニングシューズ レディース"]

# 1. 収集を作成：上位10件、google.co.jp、デスクトップ
items = [{"search_engine": "google_search", "keyword": k, "device": "desktop",
          "location": "jp", "language": "ja", "depth": 1} for k in keywords]
created = requests.post(API, json=items, headers=HEADERS).json()
ids = [row["id"] for row in created["data"]]

# 2. 待機してからSERPを取得、最長10分
serps, attempts = {}, 0
while len(serps) != len(ids) and attempts < 60:
    attempts += 1
    time.sleep(10)
    res = requests.get(API, params={"ids": ",".join(ids), "output": "json"}, headers=HEADERS).json()
    for row in res["data"]:
        if row["status"] == "done":
            urls = set()
            for block in row["results"]:
                if block["type"] == "organic":
                    urls.update(item["url"].rstrip("/") for item in block["items"])
            serps[row["keyword"]] = urls

# 3. 軸キーワードを中心にした厳密なクラスタリング
clusters, seen = [], set()
for pivot in keywords:
    if pivot in seen or pivot not in serps:
        continue
    group = [pivot]
    seen.add(pivot)
    for other in keywords:
        if other not in seen and other in serps:
            if len(serps[pivot] & serps[other]) >= THRESHOLD:
                group.append(other)
                seen.add(other)
    clusters.append(group)

for group in clusters:
    print(" | ".join(group))