# Purpose: Import libraries, define run configuration, and list the target search keywords.

from __future__ import annotations
import os, re, json, math, time, hashlib, warnings, colorsys
from collections import defaultdict
from datetime import datetime, timezone
from pathlib import Path
from urllib.parse import urlparse

warnings.filterwarnings("ignore")

import numpy as np
import pandas as pd
import requests
from bs4 import BeautifulSoup
import matplotlib.pyplot as plt
from matplotlib import cm
from IPython.display import Markdown, display

from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer
from sklearn.metrics.pairwise import cosine_similarity
from sklearn.cluster import KMeans
from sklearn.decomposition import TruncatedSVD
from sklearn.preprocessing import normalize
import networkx as nx

try:
    import hdbscan

    HDBSCAN_AVAILABLE = True
except Exception as exc:
    HDBSCAN_AVAILABLE = False
    HDBSCAN_IMPORT_ERROR = repr(exc)
try:
    import umap

    UMAP_AVAILABLE = True
except Exception as exc:
    UMAP_AVAILABLE = False
    UMAP_IMPORT_ERROR = repr(exc)

plt.rcParams["figure.dpi"] = 130
pd.set_option("display.max_colwidth", 110)

PROJECT_ROOT = Path("/root/my-portfolio")
NOTEBOOK_DIR = PROJECT_ROOT / "notebooks"
SNAPSHOT_ROOT = (
    NOTEBOOK_DIR
    / ".local"
    / "semantic-seo-lab"
    / "semantic_seo_python"
    / "20260724T010452Z"
)
SNAPSHOT_ANALYSIS = SNAPSHOT_ROOT / "analysis"
RUN_ID = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
OUTPUT_DIR = NOTEBOOK_DIR / ".local" / "semantic-seo-classical-method-baseline" / RUN_ID
FIG_DIR = OUTPUT_DIR / "figures"
OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
FIG_DIR.mkdir(parents=True, exist_ok=True)
GRAPH_EXPORTS = []
HIERARCHY_EXPORTS = []


def save_graph_png(fig, filename, title, graph_type):
    # Export every graph through one helper so PNG output is auditable.
    path = FIG_DIR / filename
    fig.savefig(path, dpi=180, bbox_inches="tight")
    GRAPH_EXPORTS.append(
        {
            "filename": filename,
            "title": title,
            "graph_type": graph_type,
            "path": str(path),
        }
    )
    return path


RUN_LIVE_ACQUISITION = False
RESULTS_PER_QUERY = 30
REQUEST_TIMEOUT = 20
USER_AGENT = (
    "Mozilla/5.0 (compatible; DecrepitFilthSemanticSeoLab/1.0; "
    "+https://decrepitfilth.art/)"
)

SEARCH_KEYWORDS = [
    "seo automation python",
    "networkx seo",
    "bert semantic seo",
    "semantic keyword clustering python",
    "nlp for seo",
    "spacy seo",
    "python for seo",
    "semantic seo nlp",
    "roberta semantic similarity seo",
    "semantic seo python",
    "python internal linking seo",
    "python content gap analysis",
]

print("run_id:", RUN_ID)
print("keywords:", len(SEARCH_KEYWORDS))
print("private output:", OUTPUT_DIR)

run_id: 20260727T230109Z
keywords: 12
private output: /root/my-portfolio/notebooks/.local/semantic-seo-classical-method-baseline/20260727T230109Z

# Purpose: Define live SERP acquisition and page extraction helpers without storing API keys.


def stable_id(value, prefix="page"):
    return (
        f"{prefix}_"
        + hashlib.sha256(str(value).encode("utf-8", errors="ignore")).hexdigest()[:16]
    )


def normalise_url(url):
    url = str(url or "").strip()
    if not url:
        return ""
    p = urlparse(url)
    return f'{p.scheme or "https"}://{p.netloc.lower()}{(p.path.rstrip("/") or "/")}'


def domain_from_url(url):
    return urlparse(str(url)).netloc.lower().replace("www.", "")


def serper_search(query, api_key, num=30):
    r = requests.post(
        "https://google.serper.dev/search",
        headers={"X-API-KEY": api_key, "Content-Type": "application/json"},
        json={"q": query, "num": num},
        timeout=REQUEST_TIMEOUT,
    )
    r.raise_for_status()
    rows = []
    for i, item in enumerate(r.json().get("organic", []), start=1):
        url = normalise_url(item.get("link") or item.get("url") or "")
        rows.append(
            {
                "query": query,
                "rank": i,
                "candidate_url": url,
                "serp_title": item.get("title", ""),
                "serp_snippet": item.get("snippet", ""),
                "page_id": stable_id(url),
            }
        )
    return rows


def extract_page(url):
    try:
        r = requests.get(
            url,
            headers={"User-Agent": USER_AGENT},
            timeout=REQUEST_TIMEOUT,
            allow_redirects=True,
        )
        html = r.text if r.ok else ""
    except Exception as exc:
        return {
            "candidate_url": url,
            "final_fetch_url": url,
            "page_id": stable_id(url),
            "fetch_status": "error",
            "error": repr(exc),
            "title": "",
            "meta_description": "",
            "h1": "",
            "headings": [],
            "clean_body": "",
        }
    soup = BeautifulSoup(html, "html.parser")
    for tag in soup(["script", "style", "noscript", "svg", "iframe"]):
        tag.decompose()
    meta = soup.find("meta", attrs={"name": re.compile("description", re.I)})
    return {
        "candidate_url": url,
        "final_fetch_url": r.url,
        "page_id": stable_id(normalise_url(url)),
        "fetch_status": "success" if html else "failed",
        "http_status": r.status_code,
        "title": soup.title.get_text(" ", strip=True) if soup.title else "",
        "meta_description": meta.get("content", "").strip() if meta else "",
        "h1": soup.find("h1").get_text(" ", strip=True) if soup.find("h1") else "",
        "headings": [
            h.get_text(" ", strip=True) for h in soup.find_all(re.compile("^h[1-3]$"))
        ][:40],
        "clean_body": re.sub(r"\s+", " ", soup.get_text(" ", strip=True))[:50000],
    }


def run_live_acquisition():
    api_key = os.getenv("SERPER_API_KEY")
    if not api_key:
        raise RuntimeError("SERPER_API_KEY is required for live acquisition.")
    hits = []
    for q in SEARCH_KEYWORDS:
        hits.extend(serper_search(q, api_key, RESULTS_PER_QUERY))
        time.sleep(0.2)
    hits = pd.DataFrame(hits).drop_duplicates(["query", "candidate_url"])
    pages = [extract_page(u) for u in sorted(hits.candidate_url.dropna().unique())]
    return hits, pd.DataFrame(pages)


print("acquisition functions ready")

acquisition functions ready

# Purpose: Load the frozen corpus or run live acquisition, then assemble page-level analysis text.


def load_snapshot(root):
    hits = pd.read_csv(root / "query_hits.csv")
    rows = [
        json.loads(line)
        for line in (root / "pages_extracted.jsonl")
        .read_text(encoding="utf-8")
        .splitlines()
        if line.strip()
    ]
    return hits, pd.DataFrame(rows)


if RUN_LIVE_ACQUISITION:
    query_hits, pages_raw = run_live_acquisition()
    source = "live acquisition"
else:
    query_hits, pages_raw = load_snapshot(SNAPSHOT_ROOT)
    source = str(SNAPSHOT_ROOT)

pages = pages_raw.copy()
if "fetch_status" in pages.columns:
    pages = pages[pages.fetch_status.astype(str).eq("success")].copy()
for col in ["title", "meta_description", "h1", "clean_body"]:
    pages[col] = pages.get(col, "").fillna("").astype(str)
pages["headings"] = pages.get("headings", [[] for _ in range(len(pages))])
pages["url"] = (
    pages.get("resolved_identity_url", pages.get("candidate_url", ""))
    .fillna("")
    .astype(str)
)
pages["domain"] = pages.url.map(domain_from_url)


def normalise_heading_text(value):
    if isinstance(value, list):
        parts = []
        for item in value:
            if isinstance(item, dict):
                parts.append(
                    str(
                        item.get("text")
                        or item.get("heading")
                        or item.get("label")
                        or " ".join(map(str, item.values()))
                    )
                )
            else:
                parts.append(str(item))
        return " ".join(parts)
    if isinstance(value, dict):
        return str(
            value.get("text")
            or value.get("heading")
            or value.get("label")
            or " ".join(map(str, value.values()))
        )
    return str(value)


pages["heading_text"] = pages.headings.apply(normalise_heading_text)
pages["document_text"] = (
    pages.title
    + ". "
    + pages.meta_description
    + ". "
    + pages.h1
    + ". "
    + pages.heading_text
    + ". "
    + pages.clean_body.str.slice(0, 6000)
)
pages = pages.drop_duplicates("page_id").reset_index(drop=True)
query_map = (
    query_hits.groupby("page_id")["query"]
    .apply(lambda s: " | ".join(sorted(set(map(str, s)))))
    .to_dict()
)
rank_map = (
    pd.to_numeric(query_hits["rank"], errors="coerce")
    .groupby(query_hits.page_id)
    .min()
    .to_dict()
)
pages["queries"] = pages.page_id.map(query_map).fillna("")
pages["best_rank"] = pages.page_id.map(rank_map)
summary = {
    "source": source,
    "queries": len(SEARCH_KEYWORDS),
    "serp_hits": len(query_hits),
    "unique_serp_pages": query_hits.page_id.nunique(),
    "extracted_pages": len(pages),
    "domains": pages.domain.nunique(),
}
display(
    Markdown(
        f"**Corpus loaded:** {summary['extracted_pages']} extracted pages "
        f"from {summary['domains']} domains, collected from "
        f"{summary['queries']} Semantic SEO/Python search queries."
    )
)

<IPython.core.display.Markdown object>

# Purpose: Visualise the acquisition summary so the sample size is inspectable at a glance.

fig, ax = plt.subplots(figsize=(8, 3.3))
labels = ["SERP hits", "Unique SERP pages", "Extracted pages", "Domains"]
vals = [
    summary["serp_hits"],
    summary["unique_serp_pages"],
    summary["extracted_pages"],
    summary["domains"],
]
ax.bar(labels, vals, color=["#6aaed6", "#77c6a3", "#f2b66d", "#c98cc9"])
for i, v in enumerate(vals):
    ax.text(i, v, str(v), ha="center", va="bottom", fontsize=9)
ax.set_title("Corpus used for the classical Semantic SEO Python baseline")
ax.set_ylabel("count")
ax.spines[["top", "right"]].set_visible(False)
fig.autofmt_xdate(rotation=12)
save_graph_png(
    fig, "01_acquisition_summary.png", ax.get_title(), "acquisition_bar_chart"
)
plt.show()
display(
    Markdown(
        "*The acquisition chart establishes the scale of the evidence base before "
        "any topic model is interpreted. The scrape rate and domain count are "
        "part of the SEO evidence, not background trivia.*"
    )
)

text output

<IPython.core.display.Markdown object>

# Purpose: Create shared text preprocessing, labelling, graph, clustering, and plotting helpers.

STOPWORDS = set("""
    about above after again against all also am an and any are as at be because
    been before being below between both but by can could did do does doing down
    during each few for from further had has have having he her here hers herself
    him himself his how i if in into is it its itself just me more most my myself
    no nor not now of off on once only or other our ours ourselves out over own
    same she should so some such than that the their theirs them themselves then
    there these they this those through to too under until up very was we were
    what when where which while who whom why will with you your yours yourself
    yourselves
    """.split())
DOMAIN_STOPWORDS = STOPWORDS | set("""
    seo search google page pages website websites content blog post guide learn
    use using used users user data tools tool free best online example examples
    need make makes like get one two way ways work works working click read
    article articles step steps help helps service services business digital
    marketing agency expert company home contact privacy terms cookies report
    reports
    """.split())


def clean_text(text):
    text = str(text).lower()
    text = re.sub(r"https?://\S+", " ", text)
    text = re.sub(r"[^a-z0-9\s\-]", " ", text)
    return re.sub(r"\s+", " ", text).strip()


pages["clean_for_model"] = pages.document_text.map(clean_text)
vectorizer = TfidfVectorizer(
    stop_words=sorted(DOMAIN_STOPWORDS),
    ngram_range=(1, 3),
    min_df=2,
    max_df=0.72,
    max_features=4500,
)
X = vectorizer.fit_transform(pages.clean_for_model)
terms = np.array(vectorizer.get_feature_names_out())
print("TF-IDF matrix:", X.shape)


def top_terms(indices, matrix=X, feature_names=terms, n=10):
    if len(indices) == 0:
        return []
    scores = np.asarray(matrix[indices].mean(axis=0)).ravel()
    order = scores.argsort()[::-1]
    return [feature_names[i] for i in order[:n] if scores[i] > 0]


def ctfidf_labels(labels, docs, n=10):
    labels = np.asarray(labels)
    groups = []
    ids = []
    for cid in sorted(set(labels)):
        if cid == -1:
            continue
        idx = np.where(labels == cid)[0]
        ids.append(int(cid))
        groups.append(" ".join(docs[i] for i in idx))
    if not groups:
        return {}
    cv = CountVectorizer(
        stop_words=sorted(DOMAIN_STOPWORDS),
        ngram_range=(1, 3),
        min_df=1,
        max_features=4000,
    )
    counts = cv.fit_transform(groups).astype(float)
    vocab = np.array(cv.get_feature_names_out())
    tf = counts.multiply(1 / np.maximum(counts.sum(axis=1), 1))
    df = np.asarray((counts > 0).sum(axis=0)).ravel()
    scores = tf.multiply(np.log(1 + len(groups) / np.maximum(df, 1))).tocsr()
    out = {}
    for r, cid in enumerate(ids):
        arr = np.asarray(scores[r].todense()).ravel()
        order = arr.argsort()[::-1]
        out[cid] = [vocab[i] for i in order[:n] if arr[i] > 0]
    return out


def page_graph(matrix, top_k=7, min_sim=0.11):
    sim = cosine_similarity(matrix)
    G = nx.Graph()
    for i, row in pages.iterrows():
        G.add_node(i, label=row.title[:90], domain=row.domain)
    for i in range(sim.shape[0]):
        for j in np.argsort(sim[i])[::-1][1 : top_k + 1]:
            if j > i and sim[i, j] >= min_sim:
                G.add_edge(i, j, weight=float(sim[i, j]))
    return G


def louvain(G):
    if G.number_of_edges() == 0:
        return np.zeros(G.number_of_nodes(), dtype=int)
    labels = np.full(G.number_of_nodes(), -1, dtype=int)
    communities = nx.algorithms.community.louvain_communities(
        G,
        weight="weight",
        seed=42,
    )
    for cid, members in enumerate(communities):
        for m in members:
            labels[m] = cid
    return labels


def summarize(labels, label_terms, method):
    rows = []
    labels = np.asarray(labels)
    for cid in sorted(set(labels)):
        idx = np.where(labels == cid)[0]
        if cid != -1:
            terms_s = " | ".join(label_terms.get(int(cid), top_terms(idx, n=8))[:8])
        else:
            terms_s = "noise / unassigned"
        if len(idx):
            reps = " || ".join(
                pages.iloc[idx]
                .sort_values("best_rank", na_position="last")
                .title.head(3)
                .tolist()
            )
        else:
            reps = ""
        rows.append(
            {
                "method": method,
                "cluster_id": int(cid),
                "page_count": len(idx),
                "domain_count": pages.iloc[idx].domain.nunique() if len(idx) else 0,
                "label_terms": terms_s,
                "representative_pages": reps,
            }
        )
    return pd.DataFrame(rows).sort_values(
        ["page_count", "domain_count"], ascending=False
    )


def split_label_terms(value, max_terms=None):
    terms_out = [term.strip() for term in str(value).split("|") if term.strip()]
    return terms_out[:max_terms] if max_terms else terms_out


def topic_candidate_name(terms):
    # Heuristic labels make the hierarchy scannable without claiming taxonomy.
    text = " ".join(terms).lower()
    rules = [
        ("Python SEO automation and scripting", ["python", "automation"]),
        ("Semantic search, entities, and intent", ["semantic", "entities"]),
        ("BERT / embedding similarity", ["bert", "similarity"]),
        ("Semantic keyword clustering", ["keyword", "clustering"]),
        ("spaCy / NLP implementation", ["spacy"]),
        ("Content gap and competitor analysis", ["gap", "competitor"]),
        ("Internal linking and site graph repair", ["internal", "link"]),
        ("NetworkX graph modelling", ["networkx", "graph"]),
        ("Natural language processing for SEO", ["natural language", "nlp"]),
        ("Machine learning for SEO analysis", ["machine learning"]),
    ]
    for label, required in rules:
        if all(part in text for part in required):
            return label
    return " / ".join(terms[:3]) if terms else "Unlabelled topic candidate"


def hierarchy_rows(summary_df, method, max_clusters=14, terms_per_cluster=8):
    rows = []
    usable = summary_df[summary_df.cluster_id != -1].head(max_clusters)
    for _, row in usable.iterrows():
        terms_for_cluster = split_label_terms(row.label_terms, terms_per_cluster)
        reps = [
            title.strip()
            for title in str(row.representative_pages).split("||")
            if title.strip()
        ][:3]
        rows.append(
            {
                "method": method,
                "cluster_id": f"C{int(row.cluster_id)}",
                "topic_candidate": topic_candidate_name(terms_for_cluster),
                "page_count": int(row.page_count),
                "domain_count": int(row.domain_count),
                "label_terms": " | ".join(terms_for_cluster),
                "representative_pages": " || ".join(reps),
            }
        )
    return pd.DataFrame(rows)


def hierarchy_markdown(hierarchy_df, method, limitation):
    lines = [
        f"### Provisional hierarchy: {method}",
        "",
        (
            "This is not a proven taxonomy. It is the hierarchy implied by this "
            "method: detected cluster -> candidate topic -> label-term evidence "
            "-> representative pages."
        ),
        "",
    ]
    for _, row in hierarchy_df.iterrows():
        terms_for_cluster = split_label_terms(row.label_terms, 8)
        reps = [
            page.strip()
            for page in str(row.representative_pages).split("||")
            if page.strip()
        ][:2]
        lines.extend(
            [
                (
                    f"- **{row.cluster_id}: {row.topic_candidate}** "
                    f"({row.page_count} pages / {row.domain_count} domains)"
                ),
                f"  - Label evidence: {', '.join(terms_for_cluster)}",
            ]
        )
        if reps:
            lines.append(f"  - Representative pages: {'; '.join(reps)}")
    lines.extend(["", f"**Method caution:** {limitation}"])
    return "\n".join(lines)


def display_topic_hierarchy(summary_df, method, limitation, filename_stub=None):
    # Display the review hierarchy without exporting per-method markdown/CSV files.
    hierarchy_df = hierarchy_rows(summary_df, method)
    markdown_text = hierarchy_markdown(hierarchy_df, method, limitation)
    display(Markdown(markdown_text))
    return hierarchy_df


METHOD_HEALTH_RECORDS = []


def method_health(labels, method):
    labels = np.asarray(labels)
    total = max(len(labels), 1)
    counts = pd.Series(labels).value_counts().sort_index()
    cluster_counts = counts[counts.index != -1]
    cluster_count = int(len(cluster_counts))
    largest_cluster_share = (
        float(cluster_counts.max() / total) if cluster_count else 0.0
    )
    noise_share = float(counts.get(-1, 0) / total)

    if largest_cluster_share > 0.50:
        method_status = "over-compressed"
    elif cluster_count < 2:
        method_status = "too few clusters"
    elif noise_share > 0.40:
        method_status = "noise-heavy"
    else:
        method_status = "usable baseline"

    return {
        "method": method,
        "cluster_count": cluster_count,
        "largest_cluster_share": round(largest_cluster_share, 3),
        "noise_share": round(noise_share, 3),
        "method_status": method_status,
    }


def register_method_health(labels, method):
    health = method_health(labels, method)
    METHOD_HEALTH_RECORDS.append(health)
    return health


def seo_interpretation_from_terms(terms):
    text = " ".join(terms).lower()
    if "internal" in text and "link" in text:
        return "Internal linking and site-graph intervention."
    if "gap" in text or "competitor" in text:
        return "Content-gap or competitor-analysis workflow."
    if "networkx" in text or "graph" in text or "edge" in text:
        return "Graph representation and relationship modelling."
    if "bert" in text or "roberta" in text or "embedding" in text:
        return "Transformer similarity / semantic representation."
    if "spacy" in text or "nlp" in text or "language" in text:
        return "NLP extraction and language-processing implementation."
    if "cluster" in text or "keyword" in text:
        return "Keyword clustering and topic grouping."
    if "automation" in text or "script" in text or "python" in text:
        return "Python SEO automation and scripting."
    return "Candidate SEO topic; requires human review."


def public_cluster_table(summary_df, max_rows=8):
    rows = []
    usable = summary_df[summary_df.cluster_id != -1].head(max_rows)
    for _, row in usable.iterrows():
        terms_for_cluster = split_label_terms(row.label_terms, 8)
        rows.append(
            {
                "method": row.method,
                "cluster_id": f"C{int(row.cluster_id)}",
                "page_count": int(row.page_count),
                "label_terms": " | ".join(terms_for_cluster),
                "SEO interpretation": seo_interpretation_from_terms(terms_for_cluster),
            }
        )
    return pd.DataFrame(rows)


def display_method_health_and_table(summary_df, labels, method, max_rows=8):
    health = register_method_health(labels, method)
    status = health["method_status"]
    display(
        Markdown(
            f"**Method health:** {health['cluster_count']} clusters; "
            f"largest cluster share {health['largest_cluster_share']:.1%}; "
            f"noise share {health['noise_share']:.1%}; status: **{status}**."
        )
    )
    table = public_cluster_table(summary_df, max_rows=max_rows)
    display(table)
    return health, table


def figure_caption(text):
    display(Markdown(f"*{text}*"))


def cluster_palette(labels):
    # Stable colours for detected communities or vector-space clusters.
    ids = sorted(set(int(x) for x in labels if int(x) != -1))
    cmap = plt.get_cmap("tab20", max(len(ids), 1))
    colours = {cid: cmap(index) for index, cid in enumerate(ids)}
    colours[-1] = (0.72, 0.72, 0.72, 0.55)
    return colours


def short_terms(value, n=4):
    return [term.strip() for term in str(value).split("|") if term.strip()][:n]


def cluster_label_map(summary_df, n=3):
    labels = {}
    for _, row in summary_df[summary_df.cluster_id != -1].iterrows():
        terms_for_cluster = short_terms(row.label_terms, n)
        label = f"C{int(row.cluster_id)}: " + ", ".join(terms_for_cluster)
        labels[int(row.cluster_id)] = label[:72]
    return labels


def plot_louvain_page_network(G, labels, summary_df, title, filename):
    # Louvain is genuinely graph-based, so show the page-similarity graph.
    # Node colour = detected community; centroid label = cluster label terms.
    labels = np.asarray(labels)
    colours = cluster_palette(labels)
    pos = nx.spring_layout(G, seed=42, weight="weight", k=0.42, iterations=140)
    fig, ax = plt.subplots(figsize=(13.5, 9.2))
    ax.set_facecolor("#fbfbfb")

    edge_widths = [max(0.25, G.edges[e].get("weight", 0.1) * 2.2) for e in G.edges]
    nx.draw_networkx_edges(
        G,
        pos,
        ax=ax,
        edge_color="#94a3b8",
        width=edge_widths,
        alpha=0.20,
    )

    node_colours = [colours.get(int(labels[node]), colours[-1]) for node in G.nodes]
    node_sizes = [34 if int(labels[node]) != -1 else 16 for node in G.nodes]
    nx.draw_networkx_nodes(
        G,
        pos,
        node_size=node_sizes,
        node_color=node_colours,
        edgecolors="white",
        linewidths=0.35,
        ax=ax,
    )

    label_terms = cluster_label_map(summary_df)
    for cid, label in label_terms.items():
        idx = [node for node in G.nodes if int(labels[node]) == cid]
        if not idx:
            continue
        xs = [pos[node][0] for node in idx]
        ys = [pos[node][1] for node in idx]
        ax.text(
            float(np.mean(xs)),
            float(np.mean(ys)),
            label,
            ha="center",
            va="center",
            fontsize=8,
            color="#111827",
            bbox=dict(
                boxstyle="round,pad=.25",
                facecolor="white",
                alpha=0.86,
                edgecolor="none",
            ),
        )

    ax.set_title(title)
    ax.axis("off")
    fig.tight_layout()
    save_graph_png(fig, filename, title, "louvain_page_graph")
    plt.show()


def plot_topic_cards(summary_df, title, filename, max_clusters=10, terms_per_cluster=6):
    # KMeans/HDBSCAN do not produce relation networks.
    # Topic cards make their cluster labels and evidence reviewable instead.
    import textwrap

    df = (
        summary_df[summary_df.cluster_id != -1]
        .head(max_clusters)
        .reset_index(drop=True)
    )
    columns = 2
    rows = math.ceil(max(len(df), 1) / columns)
    fig_height = max(4.2, rows * 3.25)
    fig, axes = plt.subplots(rows, columns, figsize=(14.5, fig_height))
    axes = np.atleast_1d(axes).ravel()
    cmap = plt.get_cmap("tab20", max(len(df), 1))

    for ax in axes:
        ax.axis("off")

    for index, (_, row) in enumerate(df.iterrows()):
        ax = axes[index]
        colour = cmap(index)
        terms_for_cluster = short_terms(row.label_terms, terms_per_cluster)
        term_line = ", ".join(terms_for_cluster)
        term_text = textwrap.fill(term_line, width=58)
        reps = [
            page.strip()
            for page in str(row.representative_pages).split("||")
            if page.strip()
        ][:2]
        evidence_lines = []
        for rep in reps:
            wrapped = textwrap.wrap(rep, width=62)
            if wrapped:
                evidence_lines.append(f"- {wrapped[0]}")
                evidence_lines.extend(f"  {line}" for line in wrapped[1:2])
        title_text = f"C{int(row.cluster_id)} · {int(row.page_count)} pages"
        body = "\n".join(
            [
                term_text,
                "",
                "Evidence:",
                *(evidence_lines if evidence_lines else ["- no representative page"]),
            ]
        )
        ax.add_patch(
            plt.Rectangle(
                (0, 0),
                1,
                1,
                transform=ax.transAxes,
                facecolor=(colour[0], colour[1], colour[2], 0.14),
                edgecolor=colour,
                linewidth=1.6,
            )
        )
        ax.text(
            0.045,
            0.89,
            title_text,
            transform=ax.transAxes,
            ha="left",
            va="top",
            fontsize=12,
            fontweight="bold",
            color="#111827",
        )
        ax.text(
            0.045,
            0.73,
            body,
            transform=ax.transAxes,
            ha="left",
            va="top",
            fontsize=10.2,
            color="#111827",
            linespacing=1.32,
        )

    fig.suptitle(title, y=0.995, fontsize=14)
    fig.tight_layout(rect=(0, 0, 1, 0.965), h_pad=0.9, w_pad=0.7)
    save_graph_png(fig, filename, title, "topic_cards")
    plt.show()


def plot_scatter(emb, labels, summary_df, title, filename):
    # Embedding methods produce point clouds, not relation networks.
    # This plot shows where each page sits in a reduced embedding space.
    emb = np.asarray(emb)
    if emb.shape[1] > 2:
        coords = TruncatedSVD(n_components=2, random_state=42).fit_transform(
            normalize(emb)
        )
    else:
        coords = emb

    labels = np.asarray(labels)
    fig, ax = plt.subplots(figsize=(10.5, 7))
    cmap = plt.get_cmap("tab20", max(len(set(labels)), 1))

    for n, cid in enumerate(sorted(set(labels))):
        mask = labels == cid
        ax.scatter(
            coords[mask, 0],
            coords[mask, 1],
            s=38,
            alpha=0.75,
            label=f"C{cid}",
            color=cmap(n),
        )

    labelled = summary_df[summary_df.cluster_id != -1].head(10)
    for _, row in labelled.iterrows():
        idx = np.where(labels == row.cluster_id)[0]
        if len(idx):
            x, y = coords[idx].mean(axis=0)
            terms = ", ".join(str(row.label_terms).split("|")[:3])
            label = f"C{int(row.cluster_id)}: {terms}"
            ax.text(
                x,
                y,
                label[:75],
                fontsize=7.5,
                ha="center",
                va="center",
                bbox=dict(
                    boxstyle="round,pad=.25",
                    facecolor="white",
                    alpha=0.72,
                    edgecolor="none",
                ),
            )

    ax.set_title(title)
    ax.spines[["top", "right"]].set_visible(False)
    save_graph_png(fig, filename, title, "embedding_scatter")
    plt.show()

TF-IDF matrix: (169, 4500)

# Purpose: Build the TF-IDF similarity graph and label Louvain communities with TF-IDF terms.

G = page_graph(X)
labels_louvain = louvain(G)
terms_louvain_tfidf = {
    int(c): top_terms(np.where(labels_louvain == c)[0], n=10)
    for c in sorted(set(labels_louvain))
    if c != -1
}
summary_louvain_tfidf = summarize(
    labels_louvain,
    terms_louvain_tfidf,
    "NetworkX-Louvain x TF-IDF",
)
print("page graph:", G.number_of_nodes(), "nodes,", G.number_of_edges(), "edges")
health_louvain_tfidf, public_louvain_tfidf = display_method_health_and_table(
    summary_louvain_tfidf,
    labels_louvain,
    "NetworkX-Louvain x TF-IDF",
)
plot_louvain_page_network(
    G,
    labels_louvain,
    summary_louvain_tfidf,
    "NetworkX-Louvain x TF-IDF: page similarity network",
    "02_louvain_tfidf_page_network.png",
)
figure_caption(
    "This is the one retained Louvain network visual. It shows the page-similarity graph "
    "that Louvain actually partitions; the labels are review aids, not proven topic names."
)
hierarchy_louvain_tfidf = display_topic_hierarchy(
    summary_louvain_tfidf,
    "NetworkX-Louvain x TF-IDF",
    "Broad discovery map, but weak co-occurrence can be mistaken for structure.",
)

stdout

page graph: 169 nodes, 528 edges

text output

<IPython.core.display.Markdown object>

html output

	method	cluster_id	page_count	label_terms	SEO interpretation
0	NetworkX-Louvain x TF-IDF	C0	31	python | automation | tasks | scripts | automate | analysis | script | keyword	Keyword clustering and topic grouping.
1	NetworkX-Louvain x TF-IDF	C11	26	semantic | ai | engines | entities | nlp | intent | keywords | entity	NLP extraction and language-processing implementation.
2	NetworkX-Louvain x TF-IDF	C9	18	bert | sentence | similarity | semantic | model | embeddings | models | sbert	Transformer similarity / semantic representation.
3	NetworkX-Louvain x TF-IDF	C7	18	clustering | keyword | keyword clustering | semantic | keywords | semantic keyword | cluster | grouping	Keyword clustering and topic grouping.
4	NetworkX-Louvain x TF-IDF	C5	17	spacy | nlp | python | text | semantic | python nlp | nltk | en	NLP extraction and language-processing implementation.
5	NetworkX-Louvain x TF-IDF	C1	16	gap | gap analysis | analysis | gaps | competitors | ai | keywords | competitor	Content-gap or competitor-analysis workflow.
6	NetworkX-Louvain x TF-IDF	C12	15	nlp | language | natural | natural language | natural language processing | language processing | processi...	NLP extraction and language-processing implementation.
7	NetworkX-Louvain x TF-IDF	C4	12	internal | links | linking | internal linking | internal links | link | site | internal linking strategy	Internal linking and site-graph intervention.

text output

<IPython.core.display.Markdown object>

text output

<IPython.core.display.Markdown object>

# Purpose: Reuse the Louvain communities and relabel them with c-TF-IDF distinctive terms.

terms_louvain_ctfidf = ctfidf_labels(
    labels_louvain,
    pages.clean_for_model.tolist(),
    n=10,
)
summary_louvain_ctfidf = summarize(
    labels_louvain,
    terms_louvain_ctfidf,
    "NetworkX-Louvain x c-TF-IDF",
)
health_louvain_ctfidf, public_louvain_ctfidf = display_method_health_and_table(
    summary_louvain_ctfidf,
    labels_louvain,
    "NetworkX-Louvain x c-TF-IDF",
)
display(
    Markdown(
        "*This variant reuses the Louvain partition above. The graph is not repeated; "
        "only the community labels change from TF-IDF to c-TF-IDF.*"
    )
)
hierarchy_louvain_ctfidf = display_topic_hierarchy(
    summary_louvain_ctfidf,
    "NetworkX-Louvain x c-TF-IDF",
    "c-TF-IDF sharpens labels, but it does not validate the Louvain partition.",
)

<IPython.core.display.Markdown object>

<IPython.core.display.Markdown object>

<IPython.core.display.Markdown object>

# Purpose: Load actual BERT/RoBERTa transformer embeddings; cached actual embeddings are allowed.

os.environ["TOKENIZERS_PARALLELISM"] = "false"
import torch
from transformers import AutoModel, AutoTokenizer
from transformers import logging as transformers_logging

transformers_logging.set_verbosity_error()
MODEL_CACHE_DIR = NOTEBOOK_DIR / ".local" / "huggingface-model-cache"
EMBEDDING_CACHE_DIR = (
    NOTEBOOK_DIR
    / ".local"
    / "semantic-seo-classical-method-baseline"
    / "embedding-cache"
)
MODEL_CACHE_DIR.mkdir(parents=True, exist_ok=True)
EMBEDDING_CACHE_DIR.mkdir(parents=True, exist_ok=True)


def page_embedding_text(row, char_limit=2400):
    # Keep the input reproducible and bounded for transformer inference.
    parts = [
        row.get("title", ""),
        row.get("description", ""),
        row.get("document_text", ""),
    ]
    text = " ".join(str(part) for part in parts if str(part).strip())
    return clean_text(text)[:char_limit]


embedding_texts = [page_embedding_text(row) for _, row in pages.iterrows()]
corpus_material = "\n".join(
    f"{row.url}	{row.title}	{text}"
    for text, row in zip(embedding_texts, pages.itertuples(index=False))
)
CORPUS_FINGERPRINT = hashlib.sha256(corpus_material.encode("utf-8")).hexdigest()[:16]
TRANSFORMER_DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
print("transformer_device:", TRANSFORMER_DEVICE)


def mean_pool(last_hidden_state, attention_mask):
    # Mean pooling converts token embeddings into one page-level vector.
    mask = attention_mask.unsqueeze(-1).expand(last_hidden_state.size()).float()
    summed = (last_hidden_state.float() * mask).sum(dim=1)
    counts = mask.sum(dim=1).clamp(min=1e-9)
    return summed / counts


def transformer_embeddings(model_name, label, batch_size=12, max_length=384):
    # Use real transformer embeddings. Do not fall back to TF-IDF/SVD here.
    safe_name = model_name.replace("/", "--")
    cache_path = EMBEDDING_CACHE_DIR / f"{safe_name}_{CORPUS_FINGERPRINT}.npy"
    meta_path = EMBEDDING_CACHE_DIR / f"{safe_name}_{CORPUS_FINGERPRINT}.json"
    if cache_path.exists() and meta_path.exists():
        arr = np.load(cache_path)
        meta = json.loads(meta_path.read_text(encoding="utf-8"))
        if arr.shape[0] == len(pages) and meta.get("model_name") == model_name:
            meta["source"] = "cached actual transformer embeddings"
            return arr, meta

    try:
        tokenizer = AutoTokenizer.from_pretrained(
            model_name,
            cache_dir=str(MODEL_CACHE_DIR),
        )
        model = AutoModel.from_pretrained(
            model_name,
            cache_dir=str(MODEL_CACHE_DIR),
        )
    except Exception as exc:
        raise RuntimeError(
            f"Actual {label} embeddings are required, but {model_name} could not load. "
            "Install/download the Hugging Face model or do not label this method as BERT/RoBERTa."
        ) from exc

    model.to(TRANSFORMER_DEVICE)
    model.eval()
    vectors = []
    with torch.inference_mode():
        for start in range(0, len(embedding_texts), batch_size):
            batch = embedding_texts[start : start + batch_size]
            encoded = tokenizer(
                batch,
                padding=True,
                truncation=True,
                max_length=max_length,
                return_tensors="pt",
            )
            encoded = {
                key: value.to(TRANSFORMER_DEVICE) for key, value in encoded.items()
            }
            output = model(**encoded)
            pooled = mean_pool(output.last_hidden_state, encoded["attention_mask"])
            vectors.append(pooled.cpu().numpy())

    arr = np.vstack(vectors).astype("float32")
    meta = {
        "label": label,
        "model_name": model_name,
        "source": "fresh actual transformer embeddings",
        "device": TRANSFORMER_DEVICE,
        "documents": int(len(embedding_texts)),
        "dimensions": int(arr.shape[1]),
        "max_length": int(max_length),
        "corpus_fingerprint": CORPUS_FINGERPRINT,
    }
    np.save(cache_path, arr)
    meta_path.write_text(json.dumps(meta, indent=2), encoding="utf-8")
    return arr, meta


bert_embeddings, bert_source = transformer_embeddings(
    "bert-base-uncased",
    "BERT",
)
roberta_embeddings, roberta_source = transformer_embeddings(
    "roberta-base",
    "RoBERTa",
)
embedding_sources_df = pd.DataFrame([bert_source, roberta_source])
display(embedding_sources_df)

transformer_device: cuda

# Purpose: Cluster BERT-style embeddings with KMeans and label the clusters with TF-IDF terms.

K = min(10, max(4, int(round(math.sqrt(len(pages))))))
labels_kmeans_bert = KMeans(
    n_clusters=K,
    random_state=42,
    n_init=20,
).fit_predict(normalize(bert_embeddings))
terms_kmeans_bert = {
    int(c): top_terms(np.where(labels_kmeans_bert == c)[0], n=10)
    for c in sorted(set(labels_kmeans_bert))
}
summary_kmeans_bert = summarize(
    labels_kmeans_bert,
    terms_kmeans_bert,
    "KMeans-BERT x TF-IDF",
)
health_kmeans_bert, public_kmeans_bert = display_method_health_and_table(
    summary_kmeans_bert,
    labels_kmeans_bert,
    "KMeans-BERT x TF-IDF",
)
plot_scatter(
    bert_embeddings,
    labels_kmeans_bert,
    summary_kmeans_bert,
    "KMeans-BERT x TF-IDF: embedding cluster map",
    "04_kmeans_bert_scatter.png",
)
figure_caption(
    "The BERT scatterplot shows how KMeans partitions actual transformer page "
    "embeddings. It is useful for seeing semantic smoothing, but KMeans still "
    "forces every page into a preselected number of clusters."
)
plot_topic_cards(
    summary_kmeans_bert,
    "KMeans-BERT x TF-IDF: topic cards",
    "05_kmeans_bert_topic_cards.png",
)
figure_caption(
    "The topic cards turn the BERT clusters into inspectable SEO evidence: label "
    "terms plus representative pages. These are candidate topics, not final "
    "editorial decisions."
)
hierarchy_kmeans_bert = display_topic_hierarchy(
    summary_kmeans_bert,
    "KMeans-BERT x TF-IDF",
    "BERT smoothing can merge SEO distinctions that matter editorially.",
)

text output

<IPython.core.display.Markdown object>

html output

	method	cluster_id	page_count	label_terms	SEO interpretation
0	KMeans-BERT x TF-IDF	C3	43	semantic | nlp | keyword | bert | intent | language | keywords | natural	Transformer similarity / semantic representation.
1	KMeans-BERT x TF-IDF	C7	28	python | automation | automate | tasks | libraries | scripts | analysis | technical	Python SEO automation and scripting.
2	KMeans-BERT x TF-IDF	C1	28	spacy | python | clustering | text | nlp | models | similarity | nltk	NLP extraction and language-processing implementation.
3	KMeans-BERT x TF-IDF	C0	26	python | graph | networkx | nodes | script | spacy | edges | link	Graph representation and relationship modelling.
4	KMeans-BERT x TF-IDF	C5	17	ai | semantic | 2026 | gap | gap analysis | claude | surfer | skill	Content-gap or competitor-analysis workflow.
5	KMeans-BERT x TF-IDF	C2	15	gap | gap analysis | analysis | gaps | competitors | keywords | competitor | ai	Content-gap or competitor-analysis workflow.
6	KMeans-BERT x TF-IDF	C9	7	internal | links | linking | internal linking | internal links | link | site | internal linking strategy	Internal linking and site-graph intervention.
7	KMeans-BERT x TF-IDF	C8	3	networkx documentation | ads | management | networkx | semantic | documentation | glossary | loading	Graph representation and relationship modelling.

text output

<IPython.core.display.Markdown object>

text output

<IPython.core.display.Markdown object>

text output

<IPython.core.display.Markdown object>

# Purpose: Cluster RoBERTa-style embeddings with KMeans and label the clusters with TF-IDF terms.

labels_kmeans_roberta = KMeans(
    n_clusters=K,
    random_state=84,
    n_init=20,
).fit_predict(normalize(roberta_embeddings))
terms_kmeans_roberta = {
    int(c): top_terms(np.where(labels_kmeans_roberta == c)[0], n=10)
    for c in sorted(set(labels_kmeans_roberta))
}
summary_kmeans_roberta = summarize(
    labels_kmeans_roberta,
    terms_kmeans_roberta,
    "KMeans-RoBERTa x TF-IDF",
)
health_kmeans_roberta, public_kmeans_roberta = display_method_health_and_table(
    summary_kmeans_roberta,
    labels_kmeans_roberta,
    "KMeans-RoBERTa x TF-IDF",
)
plot_scatter(
    roberta_embeddings,
    labels_kmeans_roberta,
    summary_kmeans_roberta,
    "KMeans-RoBERTa x TF-IDF: embedding cluster map",
    "06_kmeans_roberta_scatter.png",
)
figure_caption(
    "The RoBERTa scatterplot repeats the transformer clustering test with a "
    "second model. Agreement with BERT is stronger evidence than either model "
    "alone; disagreement is a representation warning."
)
plot_topic_cards(
    summary_kmeans_roberta,
    "KMeans-RoBERTa x TF-IDF: topic cards",
    "07_kmeans_roberta_topic_cards.png",
)
figure_caption(
    "The RoBERTa topic cards expose the label evidence for each cluster, making "
    "model-specific interpretation inspectable rather than hidden inside the "
    "embedding space."
)
hierarchy_kmeans_roberta = display_topic_hierarchy(
    summary_kmeans_roberta,
    "KMeans-RoBERTa x TF-IDF",
    "RoBERTa clusters are useful, but KMeans still imposes the cluster count.",
)

text output

<IPython.core.display.Markdown object>

html output

	method	cluster_id	page_count	label_terms	SEO interpretation
0	KMeans-RoBERTa x TF-IDF	C9	37	semantic | nlp | keyword | internal | keywords | intent | clustering | engines	NLP extraction and language-processing implementation.
1	KMeans-RoBERTa x TF-IDF	C4	31	python | spacy | nlp | clustering | semantic | text | keyword | script	NLP extraction and language-processing implementation.
2	KMeans-RoBERTa x TF-IDF	C5	30	nlp | python | language | semantic | bert | keywords | keyword | natural	Transformer similarity / semantic representation.
3	KMeans-RoBERTa x TF-IDF	C0	19	python | automation | technical | tasks | libraries | scripts | automate | reporting	Python SEO automation and scripting.
4	KMeans-RoBERTa x TF-IDF	C1	19	graph | similarity | nodes | edges | networkx | model | node | sentence	Graph representation and relationship modelling.
5	KMeans-RoBERTa x TF-IDF	C8	14	gap | gap analysis | analysis | gaps | competitors | ai | keywords | competitor	Content-gap or competitor-analysis workflow.
6	KMeans-RoBERTa x TF-IDF	C3	11	semantic | ai | networkx documentation | semantic writing | ads | unlock | management | networkx	Graph representation and relationship modelling.
7	KMeans-RoBERTa x TF-IDF	C6	6	notion | spacy | link | cleaner | link graph | internal | en | internal linking strategy	Internal linking and site-graph intervention.

text output

<IPython.core.display.Markdown object>

text output

<IPython.core.display.Markdown object>

text output

<IPython.core.display.Markdown object>

# Purpose: Run the BERTopic-style HDBSCAN workflow and label dense topic families with c-TF-IDF.

if HDBSCAN_AVAILABLE:
    base = normalize(roberta_embeddings)
    if UMAP_AVAILABLE and len(pages) > 30:
        reduced = umap.UMAP(
            n_components=8,
            n_neighbors=15,
            min_dist=0.05,
            metric="cosine",
            random_state=42,
        ).fit_transform(base)
        reduction_note = "UMAP + HDBSCAN"
    else:
        reduced = TruncatedSVD(
            n_components=min(8, base.shape[1] - 1),
            random_state=42,
        ).fit_transform(base)
        reduction_note = "SVD + HDBSCAN"
    labels_hdbscan = hdbscan.HDBSCAN(
        min_cluster_size=5,
        min_samples=2,
    ).fit_predict(reduced)
else:
    from sklearn.cluster import DBSCAN

    reduced = TruncatedSVD(
        n_components=min(8, roberta_embeddings.shape[1] - 1),
        random_state=42,
    ).fit_transform(normalize(roberta_embeddings))
    labels_hdbscan = DBSCAN(eps=0.45, min_samples=3).fit_predict(reduced)
    reduction_note = "DBSCAN fallback because HDBSCAN is unavailable"

terms_hdbscan_ctfidf = ctfidf_labels(
    labels_hdbscan,
    pages.clean_for_model.tolist(),
    n=10,
)
summary_hdbscan_ctfidf = summarize(
    labels_hdbscan,
    terms_hdbscan_ctfidf,
    "BERTopic-HDBSCAN x c-TF-IDF",
)
print(reduction_note)
health_hdbscan_ctfidf, public_hdbscan_ctfidf = display_method_health_and_table(
    summary_hdbscan_ctfidf,
    labels_hdbscan,
    "BERTopic-HDBSCAN x c-TF-IDF",
)
if health_hdbscan_ctfidf["method_status"] == "over-compressed":
    display(
        Markdown(
            "**Method failure note:** this run is over-compressed. It is kept as "
            "a blind-spot example, not treated as a successful topic model."
        )
    )
plot_scatter(
    reduced if reduced.shape[1] >= 2 else roberta_embeddings,
    labels_hdbscan,
    summary_hdbscan_ctfidf,
    "BERTopic/HDBSCAN x c-TF-IDF: dense topic families",
    "08_hdbscan_scatter.png",
)
figure_caption(
    "The HDBSCAN scatterplot tests whether dense topic families emerge without "
    "forcing every page into KMeans clusters. If the health check marks it as "
    "over-compressed, the plot is evidence of method failure rather than success."
)
plot_topic_cards(
    summary_hdbscan_ctfidf,
    "BERTopic/HDBSCAN x c-TF-IDF: topic cards",
    "09_hdbscan_topic_cards.png",
)
figure_caption(
    "These cards show what HDBSCAN retained as dense topics. A dominant card is "
    "a warning that the method collapsed too much of the SERP into one family."
)
hierarchy_hdbscan_ctfidf = display_topic_hierarchy(
    summary_hdbscan_ctfidf,
    "BERTopic-HDBSCAN x c-TF-IDF",
    "HDBSCAN can find dense topic families, but this run may over-compress them.",
)

stdout

UMAP + HDBSCAN

text output

<IPython.core.display.Markdown object>

html output

	method	cluster_id	page_count	label_terms	SEO interpretation
0	BERTopic-HDBSCAN x c-TF-IDF	C2	152	python | semantic | keyword | nlp | keywords | ai | analysis | spacy	NLP extraction and language-processing implementation.
1	BERTopic-HDBSCAN x c-TF-IDF	C0	11	ai | semantic | python | topic | started | per | explained | visibility	Python SEO automation and scripting.
2	BERTopic-HDBSCAN x c-TF-IDF	C1	6	link | spacy | web | release details | core | en | internal | details	Internal linking and site-graph intervention.

text output

<IPython.core.display.Markdown object>

text output

<IPython.core.display.Markdown object>

text output

<IPython.core.display.Markdown object>

text output

<IPython.core.display.Markdown object>

# Purpose: Compare label recurrence across all method permutations.

method_summaries = [
    summary_louvain_tfidf,
    summary_louvain_ctfidf,
    summary_kmeans_bert,
    summary_kmeans_roberta,
    summary_hdbscan_ctfidf,
]
all_summary = pd.concat(method_summaries, ignore_index=True)
all_summary.to_csv(OUTPUT_DIR / "all_method_summaries.csv", index=False)

public_cluster_summary = pd.concat(
    [
        public_louvain_tfidf,
        public_louvain_ctfidf,
        public_kmeans_bert,
        public_kmeans_roberta,
        public_hdbscan_ctfidf,
    ],
    ignore_index=True,
)
public_cluster_summary.to_csv(
    OUTPUT_DIR / "public_cluster_summary.csv",
    index=False,
)

health_df = pd.DataFrame(METHOD_HEALTH_RECORDS).drop_duplicates(
    "method",
    keep="last",
)
display(Markdown("### Method health before cross-method interpretation"))
display(health_df)

reliable_methods = health_df[
    ~health_df.method_status.isin(["over-compressed", "too few clusters"])
].method.tolist()
failed_methods = health_df[
    health_df.method_status.isin(["over-compressed", "too few clusters"])
].method.tolist()
reliable_summaries = [
    df for df in method_summaries if df.method.iloc[0] in reliable_methods
]


def label_scores(df, max_clusters=8, max_terms=8):
    scores = {}
    usable = df[df.cluster_id != -1].head(max_clusters)
    for cluster_rank, (_, row) in enumerate(usable.iterrows(), start=1):
        cluster_weight = np.log1p(float(row.get("page_count", 1))) / cluster_rank
        for term_rank, term in enumerate(
            str(row.label_terms).split("|")[:max_terms], start=1
        ):
            term = term.strip().lower()
            if len(term) > 2:
                scores[term] = scores.get(term, 0) + cluster_weight / term_rank
    return scores


method_scores = {df.method.iloc[0]: label_scores(df) for df in reliable_summaries}
method_sets = {method: set(scores) for method, scores in method_scores.items()}
aggregate_scores = defaultdict(float)
for scores in method_scores.values():
    for term, score in scores.items():
        aggregate_scores[term] += score

rows = []
for term in sorted(set().union(*method_sets.values())) if method_sets else []:
    present = [method for method, terms in method_sets.items() if term in terms]
    total_score = sum(method_scores[method].get(term, 0) for method in present)
    rows.append(
        {
            "term": term,
            "method_count": len(present),
            "total_label_score": round(total_score, 4),
            "methods": " | ".join(present),
        }
    )
term_overlap = pd.DataFrame(rows).sort_values(
    ["method_count", "total_label_score", "term"],
    ascending=[False, False, True],
)
term_overlap.to_csv(OUTPUT_DIR / "cross_method_term_overlap.csv", index=False)
recurrence_profile = (
    term_overlap.method_count.value_counts()
    .rename_axis("method_count")
    .reset_index(name="term_count")
    .sort_values("method_count", ascending=False)
)


def ranked_terms(terms, n=14):
    return sorted(
        terms,
        key=lambda term: (-aggregate_scores.get(term, 0), term),
    )[:n]


def term_line(terms):
    return ", ".join(terms) if terms else "none at this threshold"


graph_methods = [
    "NetworkX-Louvain x TF-IDF",
    "NetworkX-Louvain x c-TF-IDF",
]
transformer_methods = [
    "KMeans-BERT x TF-IDF",
    "KMeans-RoBERTa x TF-IDF",
]
graph_terms = set().union(*(method_sets.get(method, set()) for method in graph_methods))
transformer_terms = set().union(
    *(method_sets.get(method, set()) for method in transformer_methods)
)
stable_core = (
    term_overlap[term_overlap.method_count == max(len(method_sets), 1)]
    .term.head(18)
    .tolist()
)
graph_only = ranked_terms(graph_terms - transformer_terms)
transformer_only = ranked_terms(transformer_terms - graph_terms)

hdbscan_method = "BERTopic-HDBSCAN x c-TF-IDF"
hdbscan_scores = label_scores(summary_hdbscan_ctfidf)
hdbscan_terms = set(hdbscan_scores)
hdbscan_missed = ranked_terms((graph_terms | transformer_terms) - hdbscan_terms)
hdbscan_status = health_df.set_index("method").loc[hdbscan_method, "method_status"]

cross_method_lines = [
    "### Cross-method interpretation",
    "",
    (
        "**Reliable methods used for stable-core comparison:** "
        f"{', '.join(reliable_methods) or 'none'}."
    ),
    "",
    (
        "**Methods excluded from stable-core comparison:** "
        f"{', '.join(failed_methods) or 'none'}."
    ),
    "",
    f"**What appears across reliable methods:** {term_line(stable_core)}.",
    "",
    f"**What appears only when graph methods are used:** {term_line(graph_only)}.",
    "",
    (
        "**What appears only when transformer clustering is used:** "
        f"{term_line(transformer_only)}."
    ),
    "",
    (
        "**What HDBSCAN missed or over-compressed:** "
        f"status = **{hdbscan_status}**. Terms visible in other reliable "
        "methods but not retained by HDBSCAN at this threshold: "
        f"{term_line(hdbscan_missed)}."
    ),
    "",
    (
        "**Semantic SEO decision:** use the recurring terms as candidate "
        "centre-of-gravity evidence, use graph-only terms for relationship "
        "and internal-link questions, and use transformer-only terms for "
        "semantic-similarity review. Do not treat an over-compressed HDBSCAN "
        "result as a successful topic map."
    ),
]
cross_method_md = "\n".join(cross_method_lines)
display(Markdown(cross_method_md))

<IPython.core.display.Markdown object>

<IPython.core.display.Markdown object>

# Purpose: Visualise shared, majority, and method-specific vocabulary.

fig, ax = plt.subplots(figsize=(8, 4.5))
profile = recurrence_profile.sort_values("method_count")
bars = ax.barh(profile.method_count.astype(str), profile.term_count, color="#4c78a8")
ax.set_xlabel("Number of label terms")
ax.set_ylabel("Number of reliable methods in which term appears")
ax.set_title("Reliable-method recurrence profile")
for bar in bars:
    ax.text(
        bar.get_width() + 0.5,
        bar.get_y() + bar.get_height() / 2,
        int(bar.get_width()),
        va="center",
        fontsize=9,
    )
ax.spines[["top", "right"]].set_visible(False)
fig.tight_layout()
save_graph_png(
    fig,
    "10_cross_method_recurrence_profile.png",
    ax.get_title(),
    "recurrence_bar_chart",
)
plt.show()
figure_caption(
    "This chart counts vocabulary recurrence across methods that passed the "
    "health check. Failed or over-compressed methods are not allowed to define "
    "the stable core."
)

method_specific_counts = []
for method in method_sets:
    unique_terms = term_overlap[
        (term_overlap.method_count == 1) & (term_overlap.methods == method)
    ]
    method_specific_counts.append(
        {
            "method": method,
            "unique_terms": len(unique_terms),
            "examples": ", ".join(unique_terms.head(5).term.tolist()),
        }
    )

method_specific_df = pd.DataFrame(method_specific_counts).sort_values(
    "unique_terms",
    ascending=True,
)
fig, ax = plt.subplots(figsize=(9.5, 5.0))
labels = method_specific_df.method.str.replace(" x ", " × ", regex=False)
bars = ax.barh(labels, method_specific_df.unique_terms, color="#6b7280")
ax.set_xlabel("Terms appearing only in this reliable method")
ax.set_title("Reliable-method distinctiveness: method-specific label terms")
for bar, examples in zip(bars, method_specific_df.examples):
    ax.text(
        bar.get_width() + 0.35,
        bar.get_y() + bar.get_height() / 2,
        examples[:70] if examples else "none",
        va="center",
        fontsize=8,
        color="#111827",
    )
ax.spines[["top", "right"]].set_visible(False)
fig.tight_layout()
save_graph_png(
    fig,
    "11_cross_method_method_specific_terms.png",
    ax.get_title(),
    "method_specific_term_bar_chart",
)
plt.show()
figure_caption(
    "Method-specific terms show where each reliable classical method sees "
    "something the others do not. These are useful prompts for human review, "
    "not automatic content recommendations."
)

text output

<IPython.core.display.Markdown object>

text output

<IPython.core.display.Markdown object>

# Purpose: Generate the concise Codex critique of method behaviour.


def join_terms(values, n=24):
    values = list(values)[:n]
    return ", ".join(values) if values else "none at this threshold"


critique_lines = [
    "### Classical-method baseline finding",
    "",
    (
        "The reliable methods repeatedly surface Python, NLP, semantic "
        "representation, keyword clustering, automation, graph, internal-link, "
        "and content-gap language. That is enough to support the page as a "
        "Semantic SEO Python baseline."
    ),
    "",
    "### What the methods still do not decide",
    "",
    (
        "They do not decide whether a label is a real topic, whether a semantic "
        "similarity is a valid SEO relationship, or whether a content gap should "
        "become a published page. Those decisions require interpretation and "
        "human supervision."
    ),
    "",
    "### Method-specific cautions",
    "",
    (
        "- NetworkX/Louvain is useful for relationship structure, but weak "
        "edges can look meaningful."
    ),
    "- c-TF-IDF improves labels, but does not validate the community partition.",
    (
        "- BERT/RoBERTa clustering uses actual transformer embeddings, but "
        "KMeans still forces a cluster count."
    ),
    (
        "- HDBSCAN/BERTopic-style clustering is useful only if the health "
        "check avoids over-compression."
    ),
    "",
    "### Practical SEO implication",
    "",
    (
        "Use the notebook as a baseline generator and critique engine. The page "
        "earns its SEO relevance by running the classical Python methods, showing "
        "their outputs, and explaining where those outputs need supervision."
    ),
]
analysis_md = "\n".join(critique_lines)
Markdown(analysis_md)

<IPython.core.display.Markdown object>

# Purpose: Build the Part B unified semantic network from the current Part A corpus.
# The graph now consumes `pages` and `query_hits` created earlier in this notebook.
# It no longer reloads a historic external graph-output folder.

from itertools import combinations
from collections import Counter

PART_B_ANCHORS = [
    "semantic seo",
    "semantic seo python",
    "python seo",
    "python for seo",
    "seo automation",
    "seo automation python",
    "nlp",
    "natural language processing",
    "spacy",
    "bert",
    "roberta",
    "semantic similarity",
    "keyword clustering",
    "semantic keyword clustering",
    "networkx",
    "internal linking",
    "internal links",
    "content gap analysis",
    "content gaps",
    "competitor analysis",
    "content gap",
    "python internal linking",
    "python content gap analysis",
    "machine learning",
    "python libraries",
    "search results",
    "serp",
    "entities",
    "entity",
    "embeddings",
    "embedding",
    "graph",
    "graphs",
    "nodes",
    "edges",
    "topic clustering",
    "topic modelling",
    "topic modeling",
    "bertopic",
    "hdbscan",
    "louvain",
    "tf idf",
    "c tf idf",
    "rankbrain",
    "claude",
    "ahrefs",
    "screaming frog",
    "selenium",
    "automation",
]
PART_B_CANONICAL = {
    "search engine optimization": "seo",
    "search engine optimisation": "seo",
    "natural language processing": "nlp",
    "internal links": "internal linking",
    "content gaps": "content gap analysis",
    "content gap": "content gap analysis",
    "semantic keyword clustering": "keyword clustering",
    "topic modeling": "topic modelling",
    "graphs": "graph",
    "nodes": "node",
    "edges": "edge",
    "embeddings": "embedding",
    "entities": "entity",
    "python for seo": "python seo",
}
PART_B_DOMAIN_STOPWORDS = STOPWORDS | set(
    """
    seo search google page pages website websites content blog post posts guide
    guides learn use using used users user data tools tool free best online
    example examples need make makes like get one two way ways work works
    working click read article articles step steps help helps service services
    business digital marketing agency expert company home contact privacy terms
    cookies report reports result results create creating created find found want
    know subscribe share image source minute minutes min updated date author team
    chapter overview introduction conclusion beginner beginners ultimate
    comprehensive complete top today modern explained master mastering
    everything able available include includes including learn more start started
    site sites ranking rankings traffic organic optimize optimized optimisation
    optimization customer customers people
    """.split()
)
PART_B_BAD_TOKENS = PART_B_DOMAIN_STOPWORDS | set(
    """
    apr july june ago min read com www http https amp nbsp medium linkedin
    youtube video videos watch login sign cookie cookies sitemap faq faqs
    """.split()
)


def part_b_heading_text(value):
    # Keep heading extraction tolerant because scraped records can store headings
    # as strings, dictionaries, or lists depending on the extraction route.
    if isinstance(value, list):
        parts = []
        for item in value:
            if isinstance(item, dict):
                parts.append(
                    str(
                        item.get("text")
                        or item.get("heading")
                        or item.get("label")
                        or " ".join(map(str, item.values()))
                    )
                )
            else:
                parts.append(str(item))
        return " ".join(parts)
    if isinstance(value, dict):
        return str(
            value.get("text")
            or value.get("heading")
            or value.get("label")
            or " ".join(map(str, value.values()))
        )
    return str(value or "")


def part_b_clean_text(text):
    # Preserve plus/minus characters because some technical phrases use them,
    # but remove punctuation that would fragment phrase matching.
    text = str(text).lower().replace("&", " and ")
    text = re.sub(r"https?://\S+", " ", text)
    text = re.sub(r"[^a-z0-9\+\-\s]", " ", text)
    return re.sub(r"\s+", " ", text).strip()


def part_b_canonicalize(term):
    term = part_b_clean_text(term).replace("-", " ")
    term = re.sub(r"\s+", " ", term).strip()
    return PART_B_CANONICAL.get(term, term)


def part_b_valid_term(term):
    term = part_b_canonicalize(term)
    tokens = term.split()
    if len(term) < 2 or not tokens or len(tokens) > 4:
        return False
    if any(token.isdigit() for token in tokens):
        return False
    if all(token in PART_B_BAD_TOKENS for token in tokens):
        return False
    if term in PART_B_BAD_TOKENS:
        return False
    return True


def build_unified_semantic_network_from_part_a(pages_df, query_hits_df):
    # Work on copies so the Part B network does not mutate Part A objects used
    # elsewhere in the notebook.
    part_b_pages = pages_df.copy()
    part_b_queries = query_hits_df.copy()

    for column in [
        "title",
        "meta_description",
        "h1",
        "clean_body",
        "resolved_identity_url",
        "candidate_url",
        "url",
        "page_id",
    ]:
        if column not in part_b_pages.columns:
            part_b_pages[column] = ""
        part_b_pages[column] = part_b_pages[column].fillna("").astype(str)

    if "fetch_status" in part_b_pages.columns:
        part_b_pages = part_b_pages[
            part_b_pages.fetch_status.astype(str).eq("success")
        ].copy()
    part_b_pages = part_b_pages.drop_duplicates("page_id").reset_index(drop=True)

    resolved_url = part_b_pages["resolved_identity_url"]
    candidate_url = part_b_pages["candidate_url"]
    existing_url = part_b_pages["url"]
    part_b_pages["url"] = resolved_url.where(
        resolved_url.ne(""),
        candidate_url.where(candidate_url.ne(""), existing_url),
    )
    part_b_pages["domain"] = part_b_pages.url.map(domain_from_url)

    if "headings" not in part_b_pages.columns:
        part_b_pages["headings"] = [[] for _ in range(len(part_b_pages))]
    part_b_pages["heading_text"] = part_b_pages.headings.apply(part_b_heading_text)

    query_map = (
        part_b_queries.groupby("page_id")["query"]
        .apply(lambda s: " | ".join(sorted(set(map(str, s)))))
        .to_dict()
    )
    rank_map = (
        pd.to_numeric(part_b_queries["rank"], errors="coerce")
        .groupby(part_b_queries.page_id)
        .min()
        .to_dict()
    )
    part_b_pages["queries"] = part_b_pages.page_id.map(query_map).fillna("")
    part_b_pages["best_rank"] = part_b_pages.page_id.map(rank_map)

    part_b_pages["weighted_text"] = (
        (part_b_pages.title + " ") * 4
        + (part_b_pages.h1 + " ") * 4
        + (part_b_pages.meta_description + " ") * 3
        + (part_b_pages.heading_text + " ") * 3
        + part_b_pages.clean_body.str.slice(0, 12000)
    )
    part_b_pages["model_text"] = part_b_pages.weighted_text.map(part_b_clean_text)

    part_b_vectorizer = TfidfVectorizer(
        lowercase=True,
        stop_words=sorted(PART_B_DOMAIN_STOPWORDS),
        ngram_range=(1, 3),
        min_df=2,
        max_df=0.68,
        max_features=6500,
        token_pattern=r"(?u)\b[a-z][a-z0-9\+\-]{1,}\b",
    )
    part_b_X = part_b_vectorizer.fit_transform(part_b_pages.model_text)
    part_b_terms = np.array(
        [part_b_canonicalize(term) for term in part_b_vectorizer.get_feature_names_out()]
    )

    page_terms = []
    concept_page_scores = defaultdict(dict)
    concept_field_hits = defaultdict(Counter)
    anchor_patterns = {
        anchor: re.compile(r"(?<![a-z0-9])" + re.escape(anchor) + r"(?![a-z0-9])")
        for anchor in PART_B_ANCHORS
    }

    for row_number, row in part_b_pages.iterrows():
        vec = part_b_X.getrow(row_number)
        scores = defaultdict(float)
        if vec.nnz:
            order = np.argsort(vec.data)[::-1]
            for idx_pos in order[:120]:
                feature_idx = vec.indices[idx_pos]
                term = part_b_terms[feature_idx]
                raw_score = float(vec.data[idx_pos])
                if part_b_valid_term(term):
                    scores[term] = max(scores[term], raw_score)

        raw_text = part_b_clean_text(
            " ".join(
                [
                    row.title,
                    row.meta_description,
                    row.h1,
                    row.heading_text,
                    row.clean_body[:12000],
                    row.queries,
                ]
            )
        )
        for anchor, pattern in anchor_patterns.items():
            canon = part_b_canonicalize(anchor)
            if pattern.search(raw_text) and part_b_valid_term(canon):
                scores[canon] = max(scores.get(canon, 0.0), 0.36)
        for query in str(row.queries).split("|"):
            query_term = part_b_canonicalize(query)
            if part_b_valid_term(query_term):
                scores[query_term] = max(scores.get(query_term, 0.0), 0.42)

        selected = sorted(scores.items(), key=lambda item: (-item[1], item[0]))[:24]
        page_terms.append(selected)
        for term, score in selected:
            concept_page_scores[term][row.page_id] = score
            for field, value in {
                "title": row.title,
                "meta": row.meta_description,
                "h1": row.h1,
                "headings": row.heading_text,
                "queries": row.queries,
            }.items():
                if term in part_b_clean_text(value):
                    concept_field_hits[term][field] += 1

    node_rows = []
    canonical_values = set(PART_B_CANONICAL.values())
    anchor_set = set(PART_B_ANCHORS)
    for term, scores in concept_page_scores.items():
        page_ids = set(scores)
        sub = part_b_pages[part_b_pages.page_id.isin(page_ids)]
        query_support = part_b_queries[
            part_b_queries.page_id.isin(page_ids)
        ]["query"].nunique()
        best_rank = pd.to_numeric(sub.best_rank, errors="coerce").min()
        node_rows.append(
            {
                "concept": term,
                "page_support": len(page_ids),
                "domain_support": sub.domain.nunique(),
                "query_support": int(query_support),
                "tfidf_score": round(sum(scores.values()), 4),
                "best_rank": None if pd.isna(best_rank) else int(best_rank),
                "anchor": term in anchor_set or term in canonical_values,
                "field_hits": json.dumps(dict(concept_field_hits[term]), sort_keys=True),
                "example_pages": " || ".join(
                    sub.sort_values("best_rank", na_position="last")
                    .title.head(3)
                    .tolist()
                ),
                "example_urls": " || ".join(
                    sub.sort_values("best_rank", na_position="last")
                    .url.head(3)
                    .tolist()
                ),
            }
        )

    nodes_df = pd.DataFrame(node_rows)
    nodes_df["node_score"] = (
        np.log1p(nodes_df.page_support) * 2.0
        + np.log1p(nodes_df.domain_support) * 1.35
        + np.log1p(nodes_df.query_support) * 1.1
        + nodes_df.anchor.astype(float) * 2.0
        + nodes_df.tfidf_score * 0.18
    )
    nodes_df = nodes_df.sort_values(
        ["node_score", "page_support", "domain_support"],
        ascending=False,
    )
    nodes_df["concept"] = nodes_df.concept.astype(str)
    node_set_all = set(nodes_df.concept)

    edge_data = {}
    page_concept_rows = []
    for (_, row), selected in zip(part_b_pages.iterrows(), page_terms):
        selected = [(term, score) for term, score in selected if term in node_set_all]
        for term, score in selected:
            page_concept_rows.append(
                {
                    "page_id": row.page_id,
                    "title": row.title,
                    "url": row.url,
                    "domain": row.domain,
                    "queries": row.queries,
                    "best_rank": row.best_rank,
                    "concept": term,
                    "page_concept_score": round(score, 4),
                }
            )
        for (a, score_a), (b, score_b) in combinations(selected, 2):
            if a == b:
                continue
            key = tuple(sorted((a, b)))
            data = edge_data.setdefault(
                key,
                {
                    "weight": 0.0,
                    "pages": set(),
                    "domains": set(),
                    "queries": set(),
                    "examples": [],
                },
            )
            data["weight"] += math.sqrt(score_a * score_b)
            data["pages"].add(row.page_id)
            data["domains"].add(row.domain)
            for query in str(row.queries).split("|"):
                query = query.strip()
                if query:
                    data["queries"].add(query)
            if len(data["examples"]) < 4:
                data["examples"].append(row.title)

    edge_rows = []
    for (a, b), data in edge_data.items():
        edge_rows.append(
            {
                "source": str(a),
                "target": str(b),
                "weight": round(data["weight"], 4),
                "page_support": len(data["pages"]),
                "domain_support": len(data["domains"]),
                "query_support": len(data["queries"]),
                "queries": " | ".join(sorted(data["queries"])[:8]),
                "example_pages": " || ".join(data["examples"]),
            }
        )
    edges_df = pd.DataFrame(edge_rows).sort_values(
        ["page_support", "domain_support", "weight"],
        ascending=False,
    )
    edges_df["source"] = edges_df.source.astype(str)
    edges_df["target"] = edges_df.target.astype(str)

    full_nodes = nodes_df[(nodes_df.page_support >= 2) | nodes_df.anchor].copy()
    full_node_set = set(full_nodes.concept)
    edge_threshold = edges_df.weight.quantile(0.90) if len(edges_df) else 0
    full_edges = edges_df[
        edges_df.source.isin(full_node_set)
        & edges_df.target.isin(full_node_set)
        & ((edges_df.page_support >= 2) | (edges_df.weight >= edge_threshold))
    ].copy()

    full_graph = nx.Graph()
    for _, record in full_nodes.iterrows():
        full_graph.add_node(
            str(record.concept),
            label=str(record.concept),
            page_support=int(record.page_support),
            domain_support=int(record.domain_support),
            query_support=int(record.query_support),
            node_score=float(record.node_score),
            anchor=bool(record.anchor),
            best_rank=-1 if pd.isna(record.best_rank) else int(record.best_rank),
            example_pages=str(record.example_pages)[:500],
        )
    for _, record in full_edges.iterrows():
        full_graph.add_edge(
            str(record.source),
            str(record.target),
            weight=float(record.weight),
            page_support=int(record.page_support),
            domain_support=int(record.domain_support),
            query_support=int(record.query_support),
            queries=str(record.queries)[:500],
            example_pages=str(record.example_pages)[:500],
        )

    keep_nodes = set(nodes_df.head(140).concept)
    review_edges = full_edges[
        full_edges.source.isin(keep_nodes) & full_edges.target.isin(keep_nodes)
    ].copy()
    allowed_edges = set()
    for node in keep_nodes:
        local_edges = review_edges[
            (review_edges.source == node) | (review_edges.target == node)
        ]
        for _, record in local_edges.sort_values(
            ["page_support", "weight"],
            ascending=False,
        ).head(8).iterrows():
            allowed_edges.add(tuple(sorted((record.source, record.target))))
    review_edges = review_edges[
        review_edges.apply(
            lambda record: tuple(sorted((record.source, record.target))) in allowed_edges,
            axis=1,
        )
    ].copy()
    connected_nodes = set(review_edges.source) | set(review_edges.target)
    review_nodes = nodes_df[nodes_df.concept.isin(connected_nodes)].copy()

    review_graph = nx.Graph()
    for _, record in review_nodes.iterrows():
        review_graph.add_node(
            str(record.concept),
            label=str(record.concept),
            page_support=int(record.page_support),
            domain_support=int(record.domain_support),
            query_support=int(record.query_support),
            node_score=float(record.node_score),
            anchor=bool(record.anchor),
            best_rank=-1 if pd.isna(record.best_rank) else int(record.best_rank),
            example_pages=str(record.example_pages)[:500],
        )
    for _, record in review_edges.iterrows():
        review_graph.add_edge(
            str(record.source),
            str(record.target),
            weight=float(record.weight),
            page_support=int(record.page_support),
            domain_support=int(record.domain_support),
            query_support=int(record.query_support),
            queries=str(record.queries)[:500],
            example_pages=str(record.example_pages)[:500],
        )

    if review_graph.number_of_edges():
        communities = nx.algorithms.community.louvain_communities(
            review_graph,
            weight="weight",
            seed=42,
        )
    else:
        communities = [set(review_graph.nodes)]
    community_map = {
        node: community_id
        for community_id, members in enumerate(communities)
        for node in members
    }
    nx.set_node_attributes(review_graph, community_map, "community")
    nx.set_node_attributes(
        full_graph,
        {node: community_map.get(node, -1) for node in full_graph.nodes},
        "review_community",
    )

    centrality = (
        nx.betweenness_centrality(review_graph, weight="weight", normalized=True)
        if review_graph.number_of_edges()
        else {}
    )
    weighted_degree = dict(review_graph.degree(weight="weight"))
    nx.set_node_attributes(review_graph, centrality, "betweenness")
    nx.set_node_attributes(review_graph, weighted_degree, "weighted_degree")

    review_node_rows = []
    for node, attrs in review_graph.nodes(data=True):
        review_node_rows.append(
            {
                "concept": node,
                **attrs,
                "betweenness": round(float(centrality.get(node, 0)), 6),
                "weighted_degree": round(float(weighted_degree.get(node, 0)), 4),
            }
        )
    review_nodes_df = pd.DataFrame(review_node_rows).sort_values(
        ["betweenness", "weighted_degree", "page_support"],
        ascending=False,
    )
    review_edges_df = pd.DataFrame(
        [
            {"source": str(a), "target": str(b), **attrs}
            for a, b, attrs in review_graph.edges(data=True)
        ]
    ).sort_values(["page_support", "weight"], ascending=False)
    page_concept_df = pd.DataFrame(page_concept_rows)

    community_rows = []
    for community_id in sorted(set(community_map.values())):
        members = [node for node, cid in community_map.items() if cid == community_id]
        subnodes = review_nodes_df[review_nodes_df.concept.isin(members)].sort_values(
            ["node_score", "page_support"],
            ascending=False,
        )
        community_rows.append(
            {
                "community": int(community_id),
                "nodes": len(members),
                "top_concepts": " | ".join(subnodes.concept.head(12).tolist()),
                "page_support_total": int(subnodes.page_support.sum()),
                "domain_support_max": int(subnodes.domain_support.max())
                if len(subnodes)
                else 0,
            }
        )
    community_df = pd.DataFrame(community_rows).sort_values(
        ["nodes", "page_support_total"],
        ascending=False,
    )

    summary_record = {
        "pages_extracted": int(len(part_b_pages)),
        "serp_hits": int(len(part_b_queries)),
        "all_concept_candidates": int(len(nodes_df)),
        "full_graph_nodes": int(full_graph.number_of_nodes()),
        "full_graph_edges": int(full_graph.number_of_edges()),
        "review_graph_nodes": int(review_graph.number_of_nodes()),
        "review_graph_edges": int(review_graph.number_of_edges()),
        "review_communities": int(len(set(community_map.values()))),
        "top_bridge_nodes": review_nodes_df[
            [
                "concept",
                "betweenness",
                "weighted_degree",
                "page_support",
                "domain_support",
                "community",
            ]
        ]
        .head(20)
        .to_dict("records"),
    }

    return {
        "pages": part_b_pages,
        "nodes": nodes_df,
        "edges": edges_df,
        "full_nodes": full_nodes,
        "full_edges": full_edges,
        "review_nodes": review_nodes_df,
        "review_edges": review_edges_df,
        "page_concept_evidence": page_concept_df,
        "communities": community_df,
        "full_graph": full_graph,
        "review_graph": review_graph,
        "summary": summary_record,
    }


def plot_part_b_review_graph(review_graph, review_nodes_df):
    # Draw only the pruned review graph so the figure is readable enough for
    # supervision. The full graph remains available in memory for inspection.
    fig, ax = plt.subplots(figsize=(18, 14))
    pos = nx.spring_layout(
        review_graph,
        seed=42,
        weight="weight",
        k=0.55,
        iterations=180,
    )
    communities = sorted(
        {review_graph.nodes[node].get("community", -1) for node in review_graph.nodes}
    )
    cmap = plt.get_cmap("tab20", max(len(communities), 1))
    colour_for = {community: cmap(i) for i, community in enumerate(communities)}
    node_colours = [
        colour_for.get(review_graph.nodes[node].get("community", -1), (0.7, 0.7, 0.7, 1))
        for node in review_graph.nodes
    ]
    node_sizes = [
        80 + min(850, review_graph.nodes[node].get("page_support", 1) * 28)
        for node in review_graph.nodes
    ]
    edge_widths = [
        0.25 + min(3.0, review_graph.edges[edge].get("weight", 0.1) / 2.5)
        for edge in review_graph.edges
    ]
    nx.draw_networkx_edges(
        review_graph,
        pos,
        alpha=0.16,
        width=edge_widths,
        edge_color="#64748b",
        ax=ax,
    )
    nx.draw_networkx_nodes(
        review_graph,
        pos,
        node_color=node_colours,
        node_size=node_sizes,
        edgecolors="white",
        linewidths=0.5,
        ax=ax,
    )
    label_nodes = set(review_nodes_df.head(42).concept)
    labels = {node: node for node in review_graph.nodes if node in label_nodes}
    nx.draw_networkx_labels(
        review_graph,
        pos,
        labels=labels,
        font_size=8,
        font_color="#111827",
        bbox={
            "boxstyle": "round,pad=.18",
            "facecolor": "white",
            "alpha": 0.78,
            "edgecolor": "none",
        },
        ax=ax,
    )
    ax.set_title(
        "Unified semantic network from Part A scraped SERP page text and query provenance",
        fontsize=14,
    )
    ax.axis("off")
    fig.tight_layout()
    return fig


part_b_network = build_unified_semantic_network_from_part_a(pages, query_hits)
part_b_summary = part_b_network["summary"]
part_b_review_nodes_df = part_b_network["review_nodes"]
part_b_review_edges_df = part_b_network["review_edges"]
part_b_community_df = part_b_network["communities"]
part_b_review_graph = part_b_network["review_graph"]

bridge_terms = [row["concept"] for row in part_b_summary["top_bridge_nodes"][:12]]
community_lines = [
    f"- Community {int(row.community)} ({int(row.nodes)} nodes): {row.top_concepts}"
    for _, row in part_b_community_df.iterrows()
]

display(
    Markdown(
        "### Part B unified network generated from the Part A corpus\n\n"
        f"- Extracted pages: {part_b_summary['pages_extracted']}\n"
        f"- SERP hits: {part_b_summary['serp_hits']}\n"
        f"- Full graph: {part_b_summary['full_graph_nodes']} nodes / "
        f"{part_b_summary['full_graph_edges']} edges\n"
        f"- Review graph: {part_b_summary['review_graph_nodes']} nodes / "
        f"{part_b_summary['review_graph_edges']} edges\n"
        f"- Review communities: {part_b_summary['review_communities']}\n"
        f"- Leading bridge nodes: {', '.join(bridge_terms)}\n\n"
        "#### Review communities\n"
        + "\n".join(community_lines)
    )
)

part_b_fig = plot_part_b_review_graph(part_b_review_graph, part_b_review_nodes_df)
part_b_graph_path = save_graph_png(
    part_b_fig,
    "12_part_b_unified_semantic_network_review.png",
    "Part B unified semantic network from current notebook corpus",
    "part_b_unified_semantic_network",
)
plt.show()
plt.close(part_b_fig)
print("Part B graph saved to:", part_b_graph_path)

text output

<IPython.core.display.Markdown object>

stdout

Part B graph saved to: /root/my-portfolio/notebooks/.local/semantic-seo-classical-method-baseline/20260727T230109Z/figures/12_part_b_unified_semantic_network_review.png