{ "cells": [ { "cell_type": "markdown", "id": "e3bb178a", "metadata": {}, "source": [ "# Object frequency bar plot\n", "\n", "The notebook calculates a cutoff from the recurring-object frequency curve and then uses that result in the interactive graph." ] }, { "cell_type": "code", "execution_count": 1, "id": "e52a64a6-8c9a-434b-ad5f-c691f4ca40fd", "metadata": {}, "outputs": [], "source": [ "# %pip install anywidget " ] }, { "cell_type": "code", "execution_count": 2, "id": "3997c16e", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Objects: 18,254\n" ] } ], "source": [ "from pathlib import Path\n", "\n", "import pandas as pd\n", "import plotly.express as px\n", "import ipywidgets as widgets\n", "from IPython.display import display\n", "\n", "csv_path = Path(\"object_frequencies.csv\")\n", "frequency_data = pd.read_csv(csv_path).sort_values(\"rank\").reset_index(drop=True)\n", "\n", "required_columns = {\"rank\", \"object\", \"frequency\"}\n", "missing_columns = required_columns.difference(frequency_data.columns)\n", "if missing_columns:\n", " raise ValueError(f\"Missing required columns: {sorted(missing_columns)}\")\n", "\n", "print(f\"Objects: {len(frequency_data):,}\")" ] }, { "cell_type": "code", "execution_count": 3, "id": "c22a8a84", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Calculated cutoff: after rank 115\n", "Minimum frequency before cutoff: 15\n", "Objects before cutoff: 0.63%\n" ] } ], "source": [ "# Exclude singletons so their long plateau cannot determine the elbow.\n", "recurring_objects = frequency_data.loc[frequency_data[\"frequency\"] > 1].copy()\n", "\n", "# Kneedle-style detection on the normalized recurring-object curve.\n", "normalized_rank = (\n", " (recurring_objects[\"rank\"] - recurring_objects[\"rank\"].min())\n", " / (recurring_objects[\"rank\"].max() - recurring_objects[\"rank\"].min())\n", ")\n", "normalized_frequency = (\n", " (recurring_objects[\"frequency\"] - recurring_objects[\"frequency\"].min())\n", " / (recurring_objects[\"frequency\"].max() - recurring_objects[\"frequency\"].min())\n", ")\n", "distance_from_endpoint_line = (1 - normalized_rank) - normalized_frequency\n", "knee_index = distance_from_endpoint_line.idxmax()\n", "calculated_cutoff_frequency = int(frequency_data.loc[knee_index, \"frequency\"])\n", "\n", "# Include every object tied at the detected frequency.\n", "calculated_cutoff_rank = int(\n", " frequency_data.loc[\n", " frequency_data[\"frequency\"] >= calculated_cutoff_frequency, \"rank\"\n", " ].max()\n", ")\n", "calculated_cutoff_percentage = calculated_cutoff_rank / len(frequency_data) * 100\n", "\n", "frequency_data[\"cutoff_group\"] = frequency_data[\"rank\"].le(\n", " calculated_cutoff_rank\n", ").map({True: \"Before cutoff\", False: \"After cutoff\"})\n", "\n", "print(f\"Calculated cutoff: after rank {calculated_cutoff_rank:,}\")\n", "print(f\"Minimum frequency before cutoff: {calculated_cutoff_frequency}\")\n", "print(f\"Objects before cutoff: {calculated_cutoff_percentage:.2f}%\")" ] }, { "cell_type": "code", "execution_count": 4, "id": "fc5f42c3", "metadata": {}, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "ac9b0e20d85341b7a648a332ba3f9d87", "version_major": 2, "version_minor": 0 }, "text/plain": [ "interactive(children=(Dropdown(description='Display', index=2, options=(('Top 100', 100), ('Top 250', 250), ('…" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "def plot_object_frequencies(limit, log_scale):\n", " view = frequency_data.head(int(limit)).copy()\n", "\n", " figure = px.bar(\n", " view,\n", " x=\"rank\",\n", " y=\"frequency\",\n", " color=\"cutoff_group\",\n", " hover_name=\"object\",\n", " hover_data={\n", " \"rank\": True,\n", " \"frequency\": True,\n", " \"cutoff_group\": False,\n", " },\n", " color_discrete_map={\n", " \"Before cutoff\": \"#2563eb\",\n", " \"After cutoff\": \"#b8bec9\",\n", " },\n", " title=f\"Object frequency by rank: first {len(view):,} objects\",\n", " )\n", "\n", " if calculated_cutoff_rank <= len(view):\n", " figure.add_vline(\n", " x=calculated_cutoff_rank + 0.5,\n", " line_width=2,\n", " line_dash=\"dash\",\n", " line_color=\"#dc2626\",\n", " annotation_text=f\"Calculated cutoff after rank {calculated_cutoff_rank:,}\",\n", " annotation_position=\"top right\",\n", " )\n", "\n", " figure.update_layout(\n", " height=650,\n", " bargap=0,\n", " hovermode=\"closest\",\n", " legend_title_text=\"Position\",\n", " yaxis_type=\"log\" if log_scale else \"linear\",\n", " )\n", " figure.update_xaxes(title=\"Object rank\", rangeslider_visible=True)\n", " figure.update_yaxes(title=\"Frequency\")\n", " figure.show()\n", "\n", "\n", "limit_options = [\n", " (\"Top 100\", 100),\n", " (\"Top 250\", 250),\n", " (\"Top 500\", 500),\n", " (\"Top 1,000\", 1000),\n", " (f\"All objects ({len(frequency_data):,})\", len(frequency_data)),\n", "]\n", "\n", "controls = widgets.interactive(\n", " plot_object_frequencies,\n", " limit=widgets.Dropdown(\n", " options=limit_options,\n", " value=500,\n", " description=\"Display\",\n", " ),\n", " log_scale=widgets.Checkbox(\n", " value=False,\n", " description=\"Log scale\",\n", " ),\n", ")\n", "display(controls)" ] }, { "cell_type": "markdown", "id": "relation-orders-intro", "metadata": {}, "source": [ "## First- and second-order relations\n", "\n", "Relations are treated as connections in either direction. First-order objects connect directly to the analysis group. Second-order objects connect to a first-order object but are not already in the analysis group or first order." ] }, { "cell_type": "code", "execution_count": 5, "id": "calculate-relation-orders", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
groupdistinct_objects
0Analysis group115
1First order3168
2Second order4059
3Total through second order7342
\n", "
" ], "text/plain": [ " group distinct_objects\n", "0 Analysis group 115\n", "1 First order 3168\n", "2 Second order 4059\n", "3 Total through second order 7342" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "Relation instances touching the analysis group: 5,520\n" ] } ], "source": [ "import json\n", "from collections import Counter, defaultdict\n", "\n", "records_dir = Path(\"v1_1_objects_relations/records\")\n", "analysis_group = set(\n", " frequency_data.head(calculated_cutoff_rank)[\"object\"].str.strip().str.lower()\n", ")\n", "adjacency = defaultdict(set)\n", "pair_frequency = Counter()\n", "analysis_group_relation_instances = 0\n", "\n", "for record_path in records_dir.glob(\"*.json\"):\n", " record = json.loads(record_path.read_text(encoding=\"utf-8-sig\"))\n", " labels = {\n", " item[\"id\"]: item[\"label\"].strip().lower()\n", " for item in record.get(\"objects\", [])\n", " }\n", " for relation in record.get(\"relations\", []):\n", " subject = labels.get(relation.get(\"subject\"))\n", " object_ = labels.get(relation.get(\"object\"))\n", " if subject and object_ and subject != object_:\n", " pair_frequency[tuple(sorted((subject, object_)))] += 1\n", " adjacency[subject].add(object_)\n", " adjacency[object_].add(subject)\n", " if subject in analysis_group or object_ in analysis_group:\n", " analysis_group_relation_instances += 1\n", "\n", "first_order = set().union(*(adjacency[item] for item in analysis_group)) - analysis_group\n", "second_order = (\n", " set().union(*(adjacency[item] for item in first_order))\n", " - analysis_group\n", " - first_order\n", ")\n", "\n", "relation_order_summary = pd.DataFrame({\n", " \"group\": [\"Analysis group\", \"First order\", \"Second order\", \"Total through second order\"],\n", " \"distinct_objects\": [len(analysis_group), len(first_order), len(second_order),\n", " len(analysis_group | first_order | second_order)],\n", "})\n", "display(relation_order_summary)\n", "print(f\"Relation instances touching the analysis group: {analysis_group_relation_instances:,}\")" ] }, { "cell_type": "markdown", "id": "interactive-relation-network-intro", "metadata": {}, "source": [ "## Interactive relation network\n", "\n", "Zoom or pan to inspect dense areas. Click a node to highlight its directly related objects and dim unrelated objects. Use **Reset selection** to restore the complete network." ] }, { "cell_type": "code", "execution_count": 6, "id": "interactive-relation-network", "metadata": {}, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "55c7c8b0c9c04866954f97169aac74c5", "version_major": 2, "version_minor": 0 }, "text/plain": [ "VBox(children=(HBox(children=(IntSlider(value=1, continuous_update=False, description='Min pair frequency', ma…" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "import numpy as np\n", "import plotly.graph_objects as go\n", "\n", "all_network_nodes = analysis_group | first_order | second_order\n", "visible_nodes = all_network_nodes.copy()\n", "frequency_by_object = dict(zip(frequency_data[\"object\"].str.strip().str.lower(),\n", " frequency_data[\"frequency\"]))\n", "node_group = {node: \"Analysis group\" if node in analysis_group else\n", " \"First order\" if node in first_order else \"Second order\"\n", " for node in all_network_nodes}\n", "\n", "# Place analysis objects around an inner circle, then arrange their first- and\n", "# second-order neighbourhoods in corresponding outer sectors.\n", "analysis_nodes = sorted(analysis_group, key=lambda n: (-frequency_by_object.get(n, 0), n))\n", "angles = {node: 2 * np.pi * index / len(analysis_nodes)\n", " for index, node in enumerate(analysis_nodes)}\n", "positions = {node: (2 * np.cos(angles[node]), 2 * np.sin(angles[node]))\n", " for node in analysis_nodes}\n", "first_anchor = {\n", " node: max(adjacency[node] & analysis_group,\n", " key=lambda n: (frequency_by_object.get(n, 0), n))\n", " for node in first_order\n", "}\n", "second_anchor = {}\n", "for node in second_order:\n", " parent = max(adjacency[node] & first_order,\n", " key=lambda n: (frequency_by_object.get(n, 0), n))\n", " second_anchor[node] = first_anchor[parent]\n", "\n", "rng = np.random.default_rng(42)\n", "sector_width = 2 * np.pi / len(analysis_nodes)\n", "for nodes, anchors, inner_radius, outer_radius in [\n", " (sorted(first_order), first_anchor, 3.5, 7.0),\n", " (sorted(second_order), second_anchor, 7.5, 13.0),\n", "]:\n", " for node in nodes:\n", " angle = angles[anchors[node]] + rng.uniform(-0.45, 0.45) * sector_width\n", " radius = rng.uniform(inner_radius, outer_radius)\n", " positions[node] = (radius * np.cos(angle), radius * np.sin(angle))\n", "\n", "network_pair_frequency = {\n", " pair: count for pair, count in pair_frequency.items()\n", " if pair[0] in all_network_nodes and pair[1] in all_network_nodes\n", "}\n", "\n", "def network_at_threshold(threshold):\n", " pairs = [pair for pair, count in network_pair_frequency.items()\n", " if count >= threshold]\n", " nodes = analysis_group | {node for pair in pairs for node in pair}\n", " filtered_adjacency = defaultdict(set)\n", " for left, right in pairs:\n", " filtered_adjacency[left].add(right)\n", " filtered_adjacency[right].add(left)\n", " return nodes, sorted(pairs), filtered_adjacency\n", "\n", "visible_nodes, edge_pairs, current_adjacency = network_at_threshold(1)\n", "edge_x, edge_y = [], []\n", "for left, right in edge_pairs:\n", " edge_x.extend([positions[left][0], positions[right][0], None])\n", " edge_y.extend([positions[left][1], positions[right][1], None])\n", "\n", "base_edges = go.Scattergl(\n", " x=edge_x, y=edge_y, mode=\"lines\", hoverinfo=\"skip\",\n", " line=dict(width=0.35, color=\"rgba(100,116,139,0.025)\"),\n", ")\n", "highlight_edges = go.Scattergl(\n", " x=[], y=[], mode=\"lines\", hoverinfo=\"skip\",\n", " line=dict(width=0.8, color=\"rgba(71,85,105,0.18)\"),\n", ")\n", "\n", "draw_order = {\"Second order\": 0, \"First order\": 1, \"Analysis group\": 2}\n", "group_colors = {\"Analysis group\": \"#2563eb\",\n", " \"First order\": \"#10b981\",\n", " \"Second order\": \"#94a3b8\"}\n", "\n", "def node_visuals(nodes):\n", " labels = sorted(nodes, key=lambda n: (draw_order[node_group[n]],\n", " frequency_by_object.get(n, 0), n))\n", " colors = [group_colors[node_group[node]] for node in labels]\n", " opacity = [1.0 if node in analysis_group else 0.65 if node in first_order else 0.35\n", " for node in labels]\n", " frequencies = np.array([frequency_by_object.get(node, 1)\n", " for node in labels], dtype=float)\n", " log_frequencies = np.log1p(frequencies)\n", " sizes = np.interp(log_frequencies, (np.log(2), np.log1p(frequency_data[\"frequency\"].max())),\n", " (3, 42)).tolist()\n", " return labels, colors, opacity, sizes\n", "\n", "node_labels, base_colors, base_opacity, base_sizes = node_visuals(visible_nodes)\n", "node_trace = go.Scattergl(\n", " x=[positions[node][0] for node in node_labels],\n", " y=[positions[node][1] for node in node_labels],\n", " mode=\"markers\",\n", " customdata=[(node, node_group[node], frequency_by_object.get(node, 0))\n", " for node in node_labels],\n", " hovertemplate=\"%{customdata[0]}
%{customdata[1]}
Frequency: %{customdata[2]}\",\n", " marker=dict(color=base_colors, opacity=base_opacity, size=base_sizes,\n", " line=dict(width=0.7, color=\"white\")),\n", ")\n", "\n", "network_figure = go.FigureWidget(data=[base_edges, highlight_edges, node_trace])\n", "base_edges = network_figure.data[0]\n", "highlight_edges = network_figure.data[1]\n", "node_trace = network_figure.data[2]\n", "network_figure.update_layout(\n", " title=f\"Relation network: {len(visible_nodes):,} objects and {len(edge_pairs):,} object pairs\",\n", " height=850, showlegend=False, hovermode=\"closest\", dragmode=\"pan\",\n", " margin=dict(l=10, r=10, t=55, b=10),\n", " plot_bgcolor=\"white\",\n", " xaxis=dict(visible=False), yaxis=dict(visible=False, scaleanchor=\"x\", scaleratio=1),\n", ")\n", "network_figure._config = {\"scrollZoom\": True, \"displaylogo\": False}\n", "selection_text = widgets.HTML(\"Click a node to highlight its direct relations.\")\n", "reset_button = widgets.Button(description=\"Reset selection\", icon=\"refresh\")\n", "threshold_slider = widgets.IntSlider(\n", " value=1, min=1, max=4, step=1, description=\"Min pair frequency\",\n", " continuous_update=False, readout=True, style={\"description_width\": \"initial\"},\n", ")\n", "\n", "def highlight_neighbours(trace, points, state):\n", " if not points.point_inds:\n", " return\n", " selected = node_labels[points.point_inds[0]]\n", " neighbours = current_adjacency[selected]\n", " colors = [\"#dc2626\" if node == selected else\n", " \"#f59e0b\" if node in neighbours else \"#cbd5e1\"\n", " for node in node_labels]\n", " opacity = [1.0 if node == selected else 0.9 if node in neighbours else 0.06\n", " for node in node_labels]\n", " sizes = [base_sizes[index] + 8 if node == selected else\n", " base_sizes[index] + 2 if node in neighbours else base_sizes[index]\n", " for index, node in enumerate(node_labels)]\n", " hx, hy = [], []\n", " for neighbour in neighbours:\n", " hx.extend([positions[selected][0], positions[neighbour][0], None])\n", " hy.extend([positions[selected][1], positions[neighbour][1], None])\n", " with network_figure.batch_update():\n", " node_trace.marker.color = colors\n", " node_trace.marker.opacity = opacity\n", " node_trace.marker.size = sizes\n", " highlight_edges.x = hx\n", " highlight_edges.y = hy\n", " selection_text.value = (f\"{selected}: {len(neighbours):,} directly related objects\")\n", "\n", "def reset_selection(_):\n", " with network_figure.batch_update():\n", " node_trace.marker.color = base_colors\n", " node_trace.marker.opacity = base_opacity\n", " node_trace.marker.size = base_sizes\n", " highlight_edges.x = []\n", " highlight_edges.y = []\n", " selection_text.value = \"Click a node to highlight its direct relations.\"\n", "\n", "def change_threshold(change):\n", " global visible_nodes, edge_pairs, current_adjacency\n", " global node_labels, base_colors, base_opacity, base_sizes\n", " visible_nodes, edge_pairs, current_adjacency = network_at_threshold(change[\"new\"])\n", " node_labels, base_colors, base_opacity, base_sizes = node_visuals(visible_nodes)\n", " edge_x, edge_y = [], []\n", " for left, right in edge_pairs:\n", " edge_x.extend([positions[left][0], positions[right][0], None])\n", " edge_y.extend([positions[left][1], positions[right][1], None])\n", " with network_figure.batch_update():\n", " base_edges.x = edge_x\n", " base_edges.y = edge_y\n", " highlight_edges.x = []\n", " highlight_edges.y = []\n", " node_trace.x = [positions[node][0] for node in node_labels]\n", " node_trace.y = [positions[node][1] for node in node_labels]\n", " node_trace.customdata = [(node, node_group[node], frequency_by_object.get(node, 0))\n", " for node in node_labels]\n", " node_trace.marker.color = base_colors\n", " node_trace.marker.opacity = base_opacity\n", " node_trace.marker.size = base_sizes\n", " network_figure.layout.title = (\n", " f\"Relation network (pair frequency ≥ {change['new']}): \"\n", " f\"{len(visible_nodes):,} objects and {len(edge_pairs):,} object pairs\"\n", " )\n", " network_figure.layout.xaxis.autorange = True\n", " network_figure.layout.yaxis.autorange = True\n", " selection_text.value = \"Click a node to highlight its direct relations.\"\n", "\n", "node_trace.on_click(highlight_neighbours)\n", "reset_button.on_click(reset_selection)\n", "threshold_slider.observe(change_threshold, names=\"value\")\n", "display(widgets.VBox([widgets.HBox([threshold_slider, reset_button, selection_text]),\n", " network_figure]))" ] }, { "cell_type": "markdown", "id": "7f8c0970", "metadata": {}, "source": [ "## Exercise-to-exercise relation groups\n", "\n", "This section keeps direct relations where both endpoints are exercises, counts each unordered exercise pair, applies a minimum pair-frequency threshold, and finds connected components. The structural grouping is automatic; interpreting and naming the structures within a component remains a semantic judgement." ] }, { "cell_type": "code", "execution_count": 7, "id": "c6323d8a", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Exercises listed: 48\n", "Direct exercise relation instances: 124\n", "Distinct exercise pairs: 58\n", "At pair frequency ≥ 3: 14 pairs forming 5 groups\n" ] }, { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
exercise_1exercise_2pair_frequencytranscriptsrelation_types
0back squatfront squat98precedes (3), maintained capacity for (1), dif...
1humane burpeepush-up88combines (5), includes (2), is structurally im...
2bulgarian goat-bag swingkettlebell swing66regresses (2), can replace (2), prepares for (...
3goblet squathumane burpee66combines (4), includes (2)
4goblet squatoverhead squat66precedes (3), can transition to (1), can follo...
5hip thrustkettlebell swing65can replace (2), teaches position for (1), pre...
6humane burpeekettlebell swing66combines (4), includes (2)
7front squatoverhead squat55precedes (3), precedes the (1), progresses to (1)
8kettlebell snatchkettlebell swing44shares qualities with (1), contains (1), can a...
9push pressstrict press44can precede (2), can progress load beyond (1),...
10clean and jerkclean and press33differs from (1), can substitute for (1), may ...
11glute bridgehip thrust33can progress to (1), is used in (1), progresse...
12moving planksuitcase carry33is a (1), acts as (1), functions as (1)
13naked get-upturkish get-up33is an assessment version of (1), can be assess...
\n", "
" ], "text/plain": [ " exercise_1 exercise_2 pair_frequency transcripts \\\n", "0 back squat front squat 9 8 \n", "1 humane burpee push-up 8 8 \n", "2 bulgarian goat-bag swing kettlebell swing 6 6 \n", "3 goblet squat humane burpee 6 6 \n", "4 goblet squat overhead squat 6 6 \n", "5 hip thrust kettlebell swing 6 5 \n", "6 humane burpee kettlebell swing 6 6 \n", "7 front squat overhead squat 5 5 \n", "8 kettlebell snatch kettlebell swing 4 4 \n", "9 push press strict press 4 4 \n", "10 clean and jerk clean and press 3 3 \n", "11 glute bridge hip thrust 3 3 \n", "12 moving plank suitcase carry 3 3 \n", "13 naked get-up turkish get-up 3 3 \n", "\n", " relation_types \n", "0 precedes (3), maintained capacity for (1), dif... \n", "1 combines (5), includes (2), is structurally im... \n", "2 regresses (2), can replace (2), prepares for (... \n", "3 combines (4), includes (2) \n", "4 precedes (3), can transition to (1), can follo... \n", "5 can replace (2), teaches position for (1), pre... \n", "6 combines (4), includes (2) \n", "7 precedes (3), precedes the (1), progresses to (1) \n", "8 shares qualities with (1), contains (1), can a... \n", "9 can precede (2), can progress load beyond (1),... \n", "10 differs from (1), can substitute for (1), may ... \n", "11 can progress to (1), is used in (1), progresse... \n", "12 is a (1), acts as (1), functions as (1) \n", "13 is an assessment version of (1), can be assess... " ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
groupexercise_countexercises
0111back squat; bulgarian goat-bag swing; front sq...
122clean and jerk; clean and press
232moving plank; suitcase carry
342naked get-up; turkish get-up
452push press; strict press
\n", "
" ], "text/plain": [ " group exercise_count exercises\n", "0 1 11 back squat; bulgarian goat-bag swing; front sq...\n", "1 2 2 clean and jerk; clean and press\n", "2 3 2 moving plank; suitcase carry\n", "3 4 2 naked get-up; turkish get-up\n", "4 5 2 push press; strict press" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "exercise_names = {\n", " \"back squat\", \"bear-hug carry\", \"bench press\", \"bodyweight bench press\",\n", " \"bodyweight clean and press\", \"bulgarian goat-bag swing\", \"clean and jerk\",\n", " \"clean and press\", \"dead hang\", \"deadlift\", \"double-bodyweight deadlift\",\n", " \"double-kettlebell clean\", \"double-kettlebell front squat\", \"double-kettlebell press\",\n", " \"farmer walk\", \"front squat\", \"glute bridge\", \"goblet squat\",\n", " \"half-kneeling press\", \"high-repetition back squat\", \"high-repetition press\",\n", " \"high-repetition squat\", \"hip thrust\", \"humane burpee\",\n", " \"kettlebell clean and press\", \"kettlebell snatch\", \"kettlebell swing\",\n", " \"military press\", \"moving plank\", \"naked get-up\", \"neck nod\",\n", " \"overhead press\", \"overhead squat\", \"post-workout walk\", \"power clean\",\n", " \"pull-up\", \"push press\", \"push-up\", \"sandbag carry\", \"six-point rock\",\n", " \"strict press\", \"suitcase carry\", \"swimming\", \"t-y-i drill\",\n", " \"turkish get-up\", \"waiter walk\", \"walking\", \"whip snatch\",\n", "}\n", "\n", "exercise_pair_frequency = Counter()\n", "exercise_pair_relations = defaultdict(Counter)\n", "exercise_pair_transcripts = defaultdict(set)\n", "\n", "for record_path in records_dir.glob(\"*.json\"):\n", " record = json.loads(record_path.read_text(encoding=\"utf-8-sig\"))\n", " labels = {item[\"id\"]: item[\"label\"].strip().lower()\n", " for item in record.get(\"objects\", [])}\n", " for relation in record.get(\"relations\", []):\n", " subject = labels.get(relation.get(\"subject\"))\n", " object_ = labels.get(relation.get(\"object\"))\n", " if subject in exercise_names and object_ in exercise_names and subject != object_:\n", " pair = tuple(sorted((subject, object_)))\n", " exercise_pair_frequency[pair] += 1\n", " exercise_pair_relations[pair][relation.get(\"relation\", \"\")] += 1\n", " exercise_pair_transcripts[pair].add(record.get(\"video_id\", record_path.stem))\n", "\n", "def connected_exercise_groups(pair_counts, minimum_frequency):\n", " graph = defaultdict(set)\n", " for (left, right), count in pair_counts.items():\n", " if count >= minimum_frequency:\n", " graph[left].add(right)\n", " graph[right].add(left)\n", " groups, visited = [], set()\n", " for start in sorted(graph):\n", " if start in visited:\n", " continue\n", " stack, component = [start], set()\n", " visited.add(start)\n", " while stack:\n", " node = stack.pop()\n", " component.add(node)\n", " for neighbour in graph[node] - visited:\n", " visited.add(neighbour)\n", " stack.append(neighbour)\n", " groups.append(sorted(component))\n", " return sorted(groups, key=lambda group: (-len(group), group))\n", "\n", "minimum_exercise_pair_frequency = 3\n", "strong_exercise_pairs = {pair: count for pair, count in exercise_pair_frequency.items()\n", " if count >= minimum_exercise_pair_frequency}\n", "exercise_groups = connected_exercise_groups(\n", " exercise_pair_frequency, minimum_exercise_pair_frequency\n", ")\n", "\n", "pair_rows = [\n", " {\"exercise_1\": pair[0], \"exercise_2\": pair[1], \"pair_frequency\": count,\n", " \"transcripts\": len(exercise_pair_transcripts[pair]),\n", " \"relation_types\": \", \".join(\n", " f\"{name} ({frequency})\" for name, frequency in\n", " exercise_pair_relations[pair].most_common())}\n", " for pair, count in sorted(strong_exercise_pairs.items(),\n", " key=lambda item: (-item[1], item[0]))\n", "]\n", "group_rows = [{\"group\": index, \"exercise_count\": len(group),\n", " \"exercises\": \"; \".join(group)}\n", " for index, group in enumerate(exercise_groups, start=1)]\n", "\n", "print(f\"Exercises listed: {len(exercise_names):,}\")\n", "print(f\"Direct exercise relation instances: {sum(exercise_pair_frequency.values()):,}\")\n", "print(f\"Distinct exercise pairs: {len(exercise_pair_frequency):,}\")\n", "print(f\"At pair frequency ≥ {minimum_exercise_pair_frequency}: \"\n", " f\"{len(strong_exercise_pairs):,} pairs forming {len(exercise_groups):,} groups\")\n", "display(pd.DataFrame(pair_rows))\n", "display(pd.DataFrame(group_rows))" ] }, { "cell_type": "markdown", "id": "clean-slate-dashboard-heading", "metadata": {}, "source": [ "# Clean-slate searchable dashboard\n", "\n", "This section leaves the experimental workflow above intact. It independently reloads the saved object-frequency table and per-video relation records, rebuilds the indexes, and presents the test dashboard.\n", "\n", "Use **Find object** to type any part of a name. The ranked suggestions show matching objects; selecting one redraws the network as that object's direct or second-order neighbourhood. The object scope, minimum pair frequency, and depth controls recalculate the graph. The table beneath the graph shows every directed `subject → relation → object` statement without truncating rows or text." ] }, { "cell_type": "code", "execution_count": 8, "id": "clean-slate-search-dashboard", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Clean dashboard index: 18,254 objects, 20,198 relation pairs, cutoff rank 115.\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "a2cc66ac1d584f2a8f05f7bf92032c68", "version_major": 2, "version_minor": 0 }, "text/plain": [ "VBox(children=(VBox(children=(Text(value='', description='Find object', layout=Layout(width='100%'), placehold…" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "from pathlib import Path\n", "from collections import Counter, defaultdict, deque\n", "import html\n", "import json\n", "import re\n", "\n", "import numpy as np\n", "import pandas as pd\n", "import plotly.graph_objects as go\n", "import ipywidgets as widgets\n", "from IPython.display import display\n", "\n", "\n", "# Clean-slate data preparation: this section deliberately rebuilds its own\n", "# indexes from the saved CSV and per-video JSON records.\n", "dashboard_frequency_path = Path(\"object_frequencies.csv\")\n", "dashboard_records_dir = Path(\"v1_1_objects_relations/records\")\n", "\n", "dashboard_frequency_data = (\n", " pd.read_csv(dashboard_frequency_path)\n", " .sort_values(\"rank\")\n", " .reset_index(drop=True)\n", ")\n", "dashboard_frequency_data[\"object_key\"] = (\n", " dashboard_frequency_data[\"object\"].astype(str).str.strip().str.lower()\n", ")\n", "\n", "dashboard_frequency_by_object = dict(zip(\n", " dashboard_frequency_data[\"object_key\"],\n", " dashboard_frequency_data[\"frequency\"],\n", "))\n", "dashboard_rank_by_object = dict(zip(\n", " dashboard_frequency_data[\"object_key\"],\n", " dashboard_frequency_data[\"rank\"],\n", "))\n", "\n", "# Recalculate the recurring-object elbow used by the earlier workflow.\n", "dashboard_recurring = dashboard_frequency_data.loc[\n", " dashboard_frequency_data[\"frequency\"] > 1\n", "].copy()\n", "dashboard_normalized_rank = (\n", " (dashboard_recurring[\"rank\"] - dashboard_recurring[\"rank\"].min())\n", " / (dashboard_recurring[\"rank\"].max() - dashboard_recurring[\"rank\"].min())\n", ")\n", "dashboard_normalized_frequency = (\n", " (dashboard_recurring[\"frequency\"] - dashboard_recurring[\"frequency\"].min())\n", " / (dashboard_recurring[\"frequency\"].max() - dashboard_recurring[\"frequency\"].min())\n", ")\n", "dashboard_knee_index = (\n", " (1 - dashboard_normalized_rank) - dashboard_normalized_frequency\n", ").idxmax()\n", "dashboard_cutoff_frequency = int(\n", " dashboard_frequency_data.loc[dashboard_knee_index, \"frequency\"]\n", ")\n", "dashboard_cutoff_rank = int(\n", " dashboard_frequency_data.loc[\n", " dashboard_frequency_data[\"frequency\"] >= dashboard_cutoff_frequency,\n", " \"rank\",\n", " ].max()\n", ")\n", "\n", "dashboard_analysis_group = set(\n", " dashboard_frequency_data.head(dashboard_cutoff_rank)[\"object_key\"]\n", ")\n", "dashboard_pair_frequency = Counter()\n", "dashboard_triple_frequency = Counter()\n", "dashboard_triple_videos = defaultdict(set)\n", "dashboard_full_adjacency = defaultdict(set)\n", "\n", "for dashboard_record_path in dashboard_records_dir.glob(\"*.json\"):\n", " dashboard_record = json.loads(\n", " dashboard_record_path.read_text(encoding=\"utf-8-sig\")\n", " )\n", " dashboard_labels = {\n", " item[\"id\"]: item[\"label\"].strip().lower()\n", " for item in dashboard_record.get(\"objects\", [])\n", " }\n", " for dashboard_relation in dashboard_record.get(\"relations\", []):\n", " dashboard_subject = dashboard_labels.get(dashboard_relation.get(\"subject\"))\n", " dashboard_object = dashboard_labels.get(dashboard_relation.get(\"object\"))\n", " if not dashboard_subject or not dashboard_object or dashboard_subject == dashboard_object:\n", " continue\n", " dashboard_pair = tuple(sorted((dashboard_subject, dashboard_object)))\n", " dashboard_relation_text = str(\n", " dashboard_relation.get(\"relation\", \"unspecified\")\n", " ).strip()\n", " dashboard_triple = (\n", " dashboard_subject, dashboard_relation_text, dashboard_object\n", " )\n", " dashboard_video_id = dashboard_record.get(\n", " \"video_id\", dashboard_record_path.stem\n", " )\n", " dashboard_pair_frequency[dashboard_pair] += 1\n", " dashboard_triple_frequency[dashboard_triple] += 1\n", " dashboard_triple_videos[dashboard_triple].add(dashboard_video_id)\n", " dashboard_full_adjacency[dashboard_subject].add(dashboard_object)\n", " dashboard_full_adjacency[dashboard_object].add(dashboard_subject)\n", "\n", "dashboard_first_order = (\n", " set().union(*(\n", " dashboard_full_adjacency[node]\n", " for node in dashboard_analysis_group\n", " )) - dashboard_analysis_group\n", ")\n", "dashboard_second_order = (\n", " set().union(*(\n", " dashboard_full_adjacency[node]\n", " for node in dashboard_first_order\n", " ))\n", " - dashboard_analysis_group\n", " - dashboard_first_order\n", ")\n", "dashboard_relation_nodes = (\n", " dashboard_analysis_group | dashboard_first_order | dashboard_second_order\n", ")\n", "\n", "dashboard_exercise_names = {\n", " \"back squat\", \"bear-hug carry\", \"bench press\", \"bodyweight bench press\",\n", " \"bodyweight clean and press\", \"bulgarian goat-bag swing\", \"clean and jerk\",\n", " \"clean and press\", \"dead hang\", \"deadlift\", \"double-bodyweight deadlift\",\n", " \"double-kettlebell clean\", \"double-kettlebell front squat\",\n", " \"double-kettlebell press\", \"farmer walk\", \"front squat\", \"glute bridge\",\n", " \"goblet squat\", \"half-kneeling press\", \"high-repetition back squat\",\n", " \"high-repetition press\", \"high-repetition squat\", \"hip thrust\",\n", " \"humane burpee\", \"kettlebell clean and press\", \"kettlebell snatch\",\n", " \"kettlebell swing\", \"military press\", \"moving plank\", \"naked get-up\",\n", " \"neck nod\", \"overhead press\", \"overhead squat\", \"post-workout walk\",\n", " \"power clean\", \"pull-up\", \"push press\", \"push-up\", \"sandbag carry\",\n", " \"six-point rock\", \"strict press\", \"suitcase carry\", \"swimming\",\n", " \"t-y-i drill\", \"turkish get-up\", \"waiter walk\", \"walking\", \"whip snatch\",\n", "}\n", "\n", "\n", "def dashboard_node_group(node):\n", " if node in dashboard_analysis_group:\n", " return \"Analysis group\"\n", " if node in dashboard_first_order:\n", " return \"First order\"\n", " if node in dashboard_second_order:\n", " return \"Second order\"\n", " return \"Outside relation network\"\n", "\n", "\n", "dashboard_scope_nodes = {\n", " \"all\": set(dashboard_frequency_data[\"object_key\"]),\n", " \"network\": set(dashboard_relation_nodes),\n", " \"analysis\": set(dashboard_analysis_group),\n", " \"analysis_first\": set(dashboard_analysis_group | dashboard_first_order),\n", " \"exercises\": set(dashboard_exercise_names & set(dashboard_frequency_by_object)),\n", "}\n", "\n", "# Stable full-network positions. Local neighbourhoods receive a fresh radial\n", "# layout centred on the selected object.\n", "dashboard_analysis_nodes = sorted(\n", " dashboard_analysis_group,\n", " key=lambda node: (-dashboard_frequency_by_object.get(node, 0), node),\n", ")\n", "dashboard_angles = {\n", " node: 2 * np.pi * index / len(dashboard_analysis_nodes)\n", " for index, node in enumerate(dashboard_analysis_nodes)\n", "}\n", "dashboard_positions = {\n", " node: (2 * np.cos(dashboard_angles[node]), 2 * np.sin(dashboard_angles[node]))\n", " for node in dashboard_analysis_nodes\n", "}\n", "dashboard_first_anchor = {\n", " node: max(\n", " dashboard_full_adjacency[node] & dashboard_analysis_group,\n", " key=lambda neighbour: (\n", " dashboard_frequency_by_object.get(neighbour, 0), neighbour\n", " ),\n", " )\n", " for node in dashboard_first_order\n", "}\n", "dashboard_second_anchor = {}\n", "for dashboard_node in dashboard_second_order:\n", " dashboard_parents = dashboard_full_adjacency[dashboard_node] & dashboard_first_order\n", " if dashboard_parents:\n", " dashboard_parent = max(\n", " dashboard_parents,\n", " key=lambda neighbour: (\n", " dashboard_frequency_by_object.get(neighbour, 0), neighbour\n", " ),\n", " )\n", " dashboard_second_anchor[dashboard_node] = dashboard_first_anchor[dashboard_parent]\n", "\n", "dashboard_rng = np.random.default_rng(42)\n", "dashboard_sector_width = 2 * np.pi / len(dashboard_analysis_nodes)\n", "for dashboard_nodes, dashboard_anchors, dashboard_inner, dashboard_outer in [\n", " (sorted(dashboard_first_order), dashboard_first_anchor, 3.5, 7.0),\n", " (sorted(dashboard_second_anchor), dashboard_second_anchor, 7.5, 13.0),\n", "]:\n", " for dashboard_node in dashboard_nodes:\n", " dashboard_angle = (\n", " dashboard_angles[dashboard_anchors[dashboard_node]]\n", " + dashboard_rng.uniform(-0.45, 0.45) * dashboard_sector_width\n", " )\n", " dashboard_radius = dashboard_rng.uniform(dashboard_inner, dashboard_outer)\n", " dashboard_positions[dashboard_node] = (\n", " dashboard_radius * np.cos(dashboard_angle),\n", " dashboard_radius * np.sin(dashboard_angle),\n", " )\n", "\n", "print(\n", " f\"Clean dashboard index: {len(dashboard_frequency_data):,} objects, \"\n", " f\"{len(dashboard_pair_frequency):,} relation pairs, \"\n", " f\"cutoff rank {dashboard_cutoff_rank:,}.\"\n", ")\n", "\n", "\n", "# Dashboard controls.\n", "dashboard_search = widgets.Text(\n", " description=\"Find object\",\n", " placeholder=\"Type part of an object name...\",\n", " layout=widgets.Layout(width=\"100%\"),\n", " style={\"description_width\": \"90px\"},\n", ")\n", "dashboard_matches = widgets.Select(\n", " options=[],\n", " rows=8,\n", " description=\"Matches\",\n", " layout=widgets.Layout(width=\"100%\", height=\"180px\"),\n", " style={\"description_width\": \"90px\"},\n", ")\n", "dashboard_scope = widgets.Dropdown(\n", " options=[\n", " (\"All extracted objects\", \"all\"),\n", " (\"Relation-network objects\", \"network\"),\n", " (\"Analysis group\", \"analysis\"),\n", " (\"Analysis + first order\", \"analysis_first\"),\n", " (\"Exercises only\", \"exercises\"),\n", " ],\n", " value=\"network\",\n", " description=\"Object scope\",\n", " style={\"description_width\": \"initial\"},\n", ")\n", "dashboard_min_pair = widgets.IntSlider(\n", " value=3,\n", " min=1,\n", " max=4,\n", " step=1,\n", " description=\"Min pair frequency\",\n", " continuous_update=False,\n", " style={\"description_width\": \"initial\"},\n", ")\n", "dashboard_depth = widgets.ToggleButtons(\n", " options=[(\"Direct\", 1), (\"Second order\", 2)],\n", " value=1,\n", " description=\"Depth\",\n", " style={\"description_width\": \"initial\"},\n", ")\n", "dashboard_reset = widgets.Button(description=\"Reset selection\", icon=\"refresh\")\n", "dashboard_search_status = widgets.HTML()\n", "dashboard_selected_summary = widgets.HTML()\n", "dashboard_match_table = widgets.Output()\n", "dashboard_relation_table = widgets.Output()\n", "\n", "dashboard_edge_trace = go.Scattergl(\n", " x=[], y=[], mode=\"lines\", hoverinfo=\"skip\",\n", " line=dict(width=0.45, color=\"rgba(100,116,139,0.08)\"),\n", ")\n", "dashboard_selected_edge_trace = go.Scattergl(\n", " x=[], y=[], mode=\"lines\", hoverinfo=\"skip\",\n", " line=dict(width=1.4, color=\"rgba(245,158,11,0.60)\"),\n", ")\n", "dashboard_node_trace = go.Scattergl(\n", " x=[], y=[], mode=\"markers\", customdata=[],\n", " hovertemplate=(\n", " \"%{customdata[0]}
%{customdata[1]}\"\n", " \"
Frequency: %{customdata[2]}
Rank: %{customdata[3]}\"\n", " ),\n", " marker=dict(line=dict(width=0.7, color=\"white\")),\n", ")\n", "dashboard_figure = go.FigureWidget(data=[\n", " dashboard_edge_trace,\n", " dashboard_selected_edge_trace,\n", " dashboard_node_trace,\n", "])\n", "dashboard_edge_trace = dashboard_figure.data[0]\n", "dashboard_selected_edge_trace = dashboard_figure.data[1]\n", "dashboard_node_trace = dashboard_figure.data[2]\n", "dashboard_figure.update_layout(\n", " title=\"Searchable relation network\",\n", " height=780,\n", " showlegend=False,\n", " hovermode=\"closest\",\n", " dragmode=\"pan\",\n", " margin=dict(l=10, r=10, t=55, b=10),\n", " plot_bgcolor=\"white\",\n", " xaxis=dict(visible=False),\n", " yaxis=dict(visible=False, scaleanchor=\"x\", scaleratio=1),\n", ")\n", "dashboard_figure._config = {\n", " \"scrollZoom\": True,\n", " \"displaylogo\": False,\n", " \"responsive\": True,\n", "}\n", "\n", "dashboard_state = {\n", " \"selected\": None,\n", " \"node_labels\": [],\n", " \"visible_direct\": set(),\n", "}\n", "dashboard_max_local_nodes = 250\n", "dashboard_group_colors = {\n", " \"Analysis group\": \"#2563eb\",\n", " \"First order\": \"#10b981\",\n", " \"Second order\": \"#94a3b8\",\n", " \"Outside relation network\": \"#cbd5e1\",\n", "}\n", "\n", "\n", "def dashboard_filtered_pairs():\n", " allowed_nodes = dashboard_scope_nodes[dashboard_scope.value]\n", " return {\n", " pair: count\n", " for pair, count in dashboard_pair_frequency.items()\n", " if count >= dashboard_min_pair.value\n", " and pair[0] in allowed_nodes\n", " and pair[1] in allowed_nodes\n", " }\n", "\n", "\n", "def dashboard_build_adjacency(filtered_pairs):\n", " filtered_adjacency = defaultdict(set)\n", " for left, right in filtered_pairs:\n", " filtered_adjacency[left].add(right)\n", " filtered_adjacency[right].add(left)\n", " return filtered_adjacency\n", "\n", "\n", "def dashboard_search_key(node, query):\n", " if not query:\n", " return (4, -dashboard_frequency_by_object.get(node, 0), node)\n", " if node == query:\n", " match_quality = 0\n", " elif node.startswith(query):\n", " match_quality = 1\n", " elif re.search(rf\"(?:^|[\\s\\-]){re.escape(query)}\", node):\n", " match_quality = 2\n", " else:\n", " match_quality = 3\n", " return (match_quality, -dashboard_frequency_by_object.get(node, 0), node)\n", "\n", "\n", "def dashboard_update_matches(*_):\n", " query = dashboard_search.value.strip().lower()\n", " candidates = dashboard_scope_nodes[dashboard_scope.value]\n", " if query:\n", " candidates = {node for node in candidates if query in node}\n", " ranked = sorted(\n", " candidates,\n", " key=lambda node: dashboard_search_key(node, query),\n", " )[:25]\n", " dashboard_matches.options = [\n", " (\n", " f\"#{int(dashboard_rank_by_object.get(node, 0)):,} {node} \"\n", " f\"(frequency {int(dashboard_frequency_by_object.get(node, 0)):,})\",\n", " node,\n", " )\n", " for node in ranked\n", " ]\n", " dashboard_search_status.value = (\n", " f\"{len(candidates):,} text matches; showing the first \"\n", " f\"{len(ranked):,}.\"\n", " if query else\n", " f\"Showing the {len(ranked):,} most frequent objects in this scope.\"\n", " )\n", " match_rows = [\n", " {\n", " \"rank\": int(dashboard_rank_by_object.get(node, 0)),\n", " \"object\": node,\n", " \"frequency\": int(dashboard_frequency_by_object.get(node, 0)),\n", " \"network_group\": dashboard_node_group(node),\n", " \"direct_connections\": len(dashboard_full_adjacency[node]),\n", " }\n", " for node in ranked\n", " ]\n", " with dashboard_match_table:\n", " dashboard_match_table.clear_output(wait=True)\n", " display(pd.DataFrame(match_rows))\n", "\n", "\n", "def dashboard_edge_coordinates(pairs, position_map):\n", " edge_x, edge_y = [], []\n", " for left, right in pairs:\n", " if left not in position_map or right not in position_map:\n", " continue\n", " edge_x.extend([position_map[left][0], position_map[right][0], None])\n", " edge_y.extend([position_map[left][1], position_map[right][1], None])\n", " return edge_x, edge_y\n", "\n", "\n", "def dashboard_local_positions(selected, direct_nodes, second_nodes):\n", " local_positions = {selected: (0.0, 0.0)}\n", " direct_sorted = sorted(\n", " direct_nodes,\n", " key=lambda node: (\n", " -dashboard_pair_frequency[tuple(sorted((selected, node)))],\n", " -dashboard_frequency_by_object.get(node, 0),\n", " node,\n", " ),\n", " )\n", " for index, node in enumerate(direct_sorted):\n", " angle = 2 * np.pi * index / max(1, len(direct_sorted))\n", " local_positions[node] = (3.2 * np.cos(angle), 3.2 * np.sin(angle))\n", " second_sorted = sorted(\n", " second_nodes,\n", " key=lambda node: (-dashboard_frequency_by_object.get(node, 0), node),\n", " )\n", " for index, node in enumerate(second_sorted):\n", " angle = 2 * np.pi * index / max(1, len(second_sorted))\n", " local_positions[node] = (6.2 * np.cos(angle), 6.2 * np.sin(angle))\n", " return local_positions\n", "\n", "\n", "def dashboard_limit_neighbourhood(selected, adjacency):\n", " direct = set(adjacency[selected])\n", " direct_ranked = sorted(\n", " direct,\n", " key=lambda node: (\n", " -dashboard_pair_frequency[tuple(sorted((selected, node)))],\n", " -dashboard_frequency_by_object.get(node, 0),\n", " node,\n", " ),\n", " )\n", " kept_direct = set(direct_ranked[: dashboard_max_local_nodes - 1])\n", " kept_second = set()\n", " if dashboard_depth.value == 2 and len(kept_direct) < dashboard_max_local_nodes - 1:\n", " second = set().union(*(adjacency[node] for node in kept_direct))\n", " second -= kept_direct | {selected}\n", " remaining = dashboard_max_local_nodes - 1 - len(kept_direct)\n", " kept_second = set(sorted(\n", " second,\n", " key=lambda node: (-dashboard_frequency_by_object.get(node, 0), node),\n", " )[:remaining])\n", " return direct, kept_direct, kept_second\n", "\n", "\n", "def dashboard_node_sizes(nodes, selected=None):\n", " max_frequency = max(dashboard_frequency_by_object.values())\n", " sizes = []\n", " for node in nodes:\n", " frequency = max(1, dashboard_frequency_by_object.get(node, 1))\n", " size = float(np.interp(\n", " np.log1p(frequency),\n", " (np.log(2), np.log1p(max_frequency)),\n", " (5, 38),\n", " ))\n", " if node == selected:\n", " size += 10\n", " sizes.append(size)\n", " return sizes\n", "\n", "\n", "def dashboard_render_relation_table(selected, adjacency, visible_direct):\n", " visible_pairs = {\n", " tuple(sorted((selected, neighbour)))\n", " for neighbour in visible_direct\n", " }\n", " rows = []\n", " for triple, frequency in dashboard_triple_frequency.items():\n", " subject, relation, object_ = triple\n", " if tuple(sorted((subject, object_))) not in visible_pairs:\n", " continue\n", " rows.append({\n", " \"subject\": subject,\n", " \"relation\": relation,\n", " \"object\": object_,\n", " \"frequency\": frequency,\n", " \"supporting_videos\": len(dashboard_triple_videos[triple]),\n", " })\n", "\n", " with dashboard_relation_table:\n", " dashboard_relation_table.clear_output(wait=True)\n", " if rows:\n", " relation_frame = pd.DataFrame(rows)\n", " relation_frame[\"subject_total\"] = relation_frame.groupby(\n", " \"subject\"\n", " )[\"frequency\"].transform(\"sum\")\n", " relation_frame = (\n", " relation_frame\n", " .sort_values(\n", " [\"subject_total\", \"subject\", \"frequency\",\n", " \"relation\", \"object\"],\n", " ascending=[False, True, False, True, True],\n", " )\n", " .drop(columns=\"subject_total\")\n", " .set_index([\"subject\", \"relation\", \"object\"])\n", " )\n", " else:\n", " relation_frame = pd.DataFrame(\n", " columns=[\"subject\", \"relation\", \"object\",\n", " \"frequency\", \"supporting_videos\"]\n", " ).set_index([\"subject\", \"relation\", \"object\"])\n", "\n", " with pd.option_context(\n", " \"display.max_rows\", None,\n", " \"display.max_columns\", None,\n", " \"display.max_colwidth\", None,\n", " \"display.multi_sparse\", True,\n", " ):\n", " display(relation_frame)\n", "\n", "\n", "def dashboard_render_network(*_):\n", " selected = dashboard_state[\"selected\"]\n", " filtered_pairs = dashboard_filtered_pairs()\n", " filtered_adjacency = dashboard_build_adjacency(filtered_pairs)\n", " scope_nodes = dashboard_scope_nodes[dashboard_scope.value]\n", "\n", " if selected not in scope_nodes:\n", " selected = None\n", " dashboard_state[\"selected\"] = None\n", "\n", " if selected:\n", " all_direct, direct_nodes, second_nodes = dashboard_limit_neighbourhood(\n", " selected, filtered_adjacency\n", " )\n", " visible_nodes = {selected} | direct_nodes | second_nodes\n", " visible_pairs = {\n", " pair: count\n", " for pair, count in filtered_pairs.items()\n", " if pair[0] in visible_nodes and pair[1] in visible_nodes\n", " }\n", " direct_pairs = {\n", " pair: count\n", " for pair, count in visible_pairs.items()\n", " if selected in pair\n", " }\n", " base_pairs = {\n", " pair: count\n", " for pair, count in visible_pairs.items()\n", " if selected not in pair\n", " }\n", " position_map = dashboard_local_positions(\n", " selected, direct_nodes, second_nodes\n", " )\n", " node_labels = [selected] + sorted(\n", " direct_nodes,\n", " key=lambda node: (\n", " -dashboard_pair_frequency[tuple(sorted((selected, node)))], node\n", " ),\n", " ) + sorted(second_nodes)\n", " node_colors = [\n", " \"#dc2626\" if node == selected else\n", " \"#f59e0b\" if node in direct_nodes else \"#14b8a6\"\n", " for node in node_labels\n", " ]\n", " node_opacity = [1.0] * len(node_labels)\n", " omitted = max(0, len(all_direct) - len(direct_nodes))\n", " dashboard_selected_summary.value = (\n", " f\"{html.escape(selected)} — rank \"\n", " f\"{int(dashboard_rank_by_object.get(selected, 0)):,}, frequency \"\n", " f\"{int(dashboard_frequency_by_object.get(selected, 0)):,}, \"\n", " f\"{len(all_direct):,} direct relations under the active filters.\"\n", " + (\n", " f\" Display limited to the strongest {len(direct_nodes):,}; \"\n", " f\"{omitted:,} omitted.\"\n", " if omitted else \"\"\n", " )\n", " )\n", " dashboard_render_relation_table(selected, filtered_adjacency, direct_nodes)\n", " title = (\n", " f\"Neighbourhood of {selected}: {len(visible_nodes):,} objects and \"\n", " f\"{len(visible_pairs):,} pairs\"\n", " )\n", " else:\n", " visible_pairs = filtered_pairs\n", " if dashboard_scope.value in {\"all\", \"network\", \"analysis_first\"}:\n", " visible_nodes = dashboard_analysis_group | {\n", " node for pair in visible_pairs for node in pair\n", " }\n", " else:\n", " visible_nodes = {\n", " node for pair in visible_pairs for node in pair\n", " }\n", " if dashboard_scope.value == \"analysis\":\n", " visible_nodes |= dashboard_analysis_group\n", " elif dashboard_scope.value == \"exercises\":\n", " visible_nodes |= dashboard_exercise_names & scope_nodes\n", " visible_nodes &= scope_nodes\n", " node_labels = sorted(\n", " visible_nodes,\n", " key=lambda node: (\n", " {\"Second order\": 0, \"First order\": 1, \"Analysis group\": 2,\n", " \"Outside relation network\": -1}[dashboard_node_group(node)],\n", " dashboard_frequency_by_object.get(node, 0),\n", " node,\n", " ),\n", " )\n", " position_map = {\n", " node: dashboard_positions[node]\n", " for node in node_labels\n", " if node in dashboard_positions\n", " }\n", " node_labels = [node for node in node_labels if node in position_map]\n", " node_colors = [\n", " dashboard_group_colors[dashboard_node_group(node)]\n", " for node in node_labels\n", " ]\n", " node_opacity = [\n", " 1.0 if node in dashboard_analysis_group else\n", " 0.65 if node in dashboard_first_order else 0.35\n", " for node in node_labels\n", " ]\n", " base_pairs = visible_pairs\n", " direct_pairs = {}\n", " dashboard_selected_summary.value = (\n", " \"No object selected. Search above or click a node to isolate its \"\n", " \"relational neighbourhood.\"\n", " )\n", " with dashboard_relation_table:\n", " dashboard_relation_table.clear_output(wait=True)\n", " title = (\n", " f\"Filtered relation network: {len(node_labels):,} objects and \"\n", " f\"{len(visible_pairs):,} pairs\"\n", " )\n", "\n", " base_x, base_y = dashboard_edge_coordinates(base_pairs, position_map)\n", " selected_x, selected_y = dashboard_edge_coordinates(direct_pairs, position_map)\n", " dashboard_state[\"node_labels\"] = node_labels\n", " dashboard_state[\"visible_direct\"] = set(\n", " filtered_adjacency[selected]\n", " ) if selected else set()\n", "\n", " with dashboard_figure.batch_update():\n", " dashboard_edge_trace.x = base_x\n", " dashboard_edge_trace.y = base_y\n", " dashboard_selected_edge_trace.x = selected_x\n", " dashboard_selected_edge_trace.y = selected_y\n", " dashboard_node_trace.x = [position_map[node][0] for node in node_labels]\n", " dashboard_node_trace.y = [position_map[node][1] for node in node_labels]\n", " dashboard_node_trace.customdata = [\n", " (\n", " node,\n", " dashboard_node_group(node),\n", " int(dashboard_frequency_by_object.get(node, 0)),\n", " int(dashboard_rank_by_object.get(node, 0)),\n", " )\n", " for node in node_labels\n", " ]\n", " dashboard_node_trace.marker.color = node_colors\n", " dashboard_node_trace.marker.opacity = node_opacity\n", " dashboard_node_trace.marker.size = dashboard_node_sizes(\n", " node_labels, selected\n", " )\n", " dashboard_figure.layout.title = title\n", " dashboard_figure.layout.xaxis.autorange = True\n", " dashboard_figure.layout.yaxis.autorange = True\n", "\n", "\n", "def dashboard_select_object(node):\n", " if not node:\n", " return\n", " dashboard_state[\"selected\"] = node\n", " if dashboard_search.value != node:\n", " dashboard_search.value = node\n", " dashboard_render_network()\n", "\n", "\n", "def dashboard_on_match(change):\n", " if change.get(\"name\") == \"value\" and change.get(\"new\"):\n", " dashboard_select_object(change[\"new\"])\n", "\n", "\n", "def dashboard_on_node_click(trace, points, state):\n", " if not points.point_inds:\n", " return\n", " dashboard_select_object(\n", " dashboard_state[\"node_labels\"][points.point_inds[0]]\n", " )\n", "\n", "\n", "def dashboard_on_filter(change):\n", " if change.get(\"name\") != \"value\":\n", " return\n", " dashboard_update_matches()\n", " dashboard_render_network()\n", "\n", "\n", "def dashboard_reset_selection(_):\n", " dashboard_state[\"selected\"] = None\n", " dashboard_search.value = \"\"\n", " dashboard_update_matches()\n", " dashboard_render_network()\n", "\n", "\n", "dashboard_search.observe(dashboard_update_matches, names=\"value\")\n", "dashboard_matches.observe(dashboard_on_match, names=\"value\")\n", "dashboard_scope.observe(dashboard_on_filter, names=\"value\")\n", "dashboard_min_pair.observe(dashboard_on_filter, names=\"value\")\n", "dashboard_depth.observe(dashboard_on_filter, names=\"value\")\n", "dashboard_reset.on_click(dashboard_reset_selection)\n", "dashboard_node_trace.on_click(dashboard_on_node_click)\n", "\n", "dashboard_update_matches()\n", "dashboard_render_network()\n", "\n", "dashboard_controls = widgets.VBox([\n", " dashboard_search,\n", " dashboard_search_status,\n", " dashboard_matches,\n", " dashboard_match_table,\n", " widgets.HBox([dashboard_scope, dashboard_min_pair]),\n", " widgets.HBox([dashboard_depth, dashboard_reset]),\n", " dashboard_selected_summary,\n", "])\n", "\n", "display(widgets.VBox([\n", " dashboard_controls,\n", " dashboard_figure,\n", " widgets.HTML(\"

Relations for the selected object

\"),\n", " dashboard_relation_table,\n", "]))\n" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.11.15" } }, "nbformat": 4, "nbformat_minor": 5 }