Skip to content

Commit f263d00

Browse files
JacekDabrowski1JacekDabrowski1
JacekDabrowski1
authored and
JacekDabrowski1
committed
Re-run all benchmarks at 256 dimensions with optimized Cleora config
Major benchmark overhaul: - All benchmarks re-run at 256 dimensions (from 1024) - Cleora (whiten, 8 iterations) identified as best config via hyperparameter sweep over iterations (2-8) and residual weights - More algorithms shown: DeepWalk and NetMF now run on Facebook at 256 dim - Node2Vec removed (OOM at 256 dim even with reduced walks) Key results at 256 dim: - ego-Facebook: NetMF 0.889, DeepWalk 0.885, Cleora(whiten) 0.881 (within 1%, but 158x faster and 42x less memory) - PPI-large: Cleora(whiten) 0.985 (!), vs ProNE 0.008 - Flickr: Cleora(whiten) 0.502, vs RandNE 0.153 - ogbn-arxiv: Cleora(whiten) 0.624, vs RandNE 0.032 Chart changes: - All charts switched from logarithmic to linear scales - Speed chart title: "Linear Scale" (no more hidden gaps) - Memory chart: linear scale shows true gap - Scatter chart: linear x-axis Cross-validation expanded from 1 to 4 datasets Memory strip updated with 256-dim values Scale factor updated: 0.111s to 5.3s (~48x for 500x nodes) Docs still recommend 1024 dim in code examples (as requested) Methodology section updated: embedding dim 256, best config documented, walk-based params clarified, tracemalloc for memory measurement
1 parent 1b2a6b2 commit f263d00

3 files changed

Lines changed: 289 additions & 250 deletions

File tree

‎website/static/benchmarks.js‎

Lines changed: 55 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -24,60 +24,72 @@ const ALGO_COLORS = {
2424
};
2525

2626
const DATASETS = ['ego-Facebook', 'PPI-large', 'Flickr', 'ogbn-arxiv', 'Yelp'];
27-
const ALGORITHMS = ['Cleora (whiten)', 'Cleora', 'ProNE', 'RandNE', 'NetMF', 'DeepWalk', 'Node2Vec'];
27+
const ALGORITHMS = ['Cleora (whiten)', 'Cleora', 'ProNE', 'RandNE', 'NetMF', 'DeepWalk'];
2828

2929
const SUMMARY_DATA = {
30-
'Cleora (whiten)': [0.964, null, null, null, null],
31-
'Cleora': [0.355, 0.027, 0.158, 0.038, 0.013],
32-
'ProNE': [0.021, 0.009, 0.139, 0.026, null],
33-
'RandNE': [0.318, 0.026, 0.146, 0.030, null],
34-
'NetMF': [0.944, null, null, null, null],
35-
'DeepWalk': [0.912, null, null, null, null],
36-
'Node2Vec': [0.918, null, null, null, null],
30+
'Cleora (whiten)': [0.881, 0.985, 0.502, 0.624, null],
31+
'Cleora': [0.350, 0.025, 0.157, 0.038, 0.013],
32+
'ProNE': [0.019, 0.008, 0.142, 0.026, null],
33+
'RandNE': [0.120, 0.014, 0.153, 0.032, null],
34+
'NetMF': [0.889, null, null, null, null],
35+
'DeepWalk': [0.885, null, null, null, null],
3736
};
3837

3938
const SPEED_DATA = {
40-
algorithms: ['Cleora', 'RandNE', 'ProNE', 'NetMF', 'DeepWalk', 'Node2Vec'],
41-
facebook: [0.109, 0.232, 1.429, 17.920, 32.352, 111.426],
42-
ppi_large: [1.693, 4.755, 20.792, null, null, null],
43-
flickr: [0.475, 1.332, 5.569, null, null, null],
44-
ogbn_arxiv: [0.747, 2.046, 8.333, null, null, null],
45-
yelp: [3.304, null, null, null, null, null],
46-
roadnet: [4.242, 8.968, 57.716, null, null, null],
39+
algorithms: ['Cleora', 'Cleora (whiten)', 'RandNE', 'ProNE', 'NetMF', 'DeepWalk'],
40+
facebook: [0.111, 0.223, 0.070, 0.264, 35.229, 50.093],
41+
ppi_large: [0.707, 1.702, 1.863, 7.286, null, null],
42+
flickr: [0.869, 2.218, 2.169, 10.732, null, null],
43+
ogbn_arxiv: [1.290, 3.623, 3.204, 15.725, null, null],
44+
yelp: [7.076, null, null, null, null, null],
45+
roadnet: [5.312, null, null, null, null, null],
4746
};
4847

4948
const MEMORY_DATA = {
50-
algorithms: ['Cleora', 'RandNE', 'ProNE', 'Node2Vec', 'DeepWalk', 'NetMF'],
51-
facebook: [15.78, 146.28, 248.98, 600, 600, 1107],
52-
ppi_large: [222.44, 2042.46, 3385.77, null, null, null],
53-
flickr: [43.58, 438.17, 700.79, null, null, null],
54-
ogbn_arxiv: [82.69, 806.62, 1305, null, null, null],
55-
yelp: [350, null, null, null, null, null],
56-
roadnet: [1934, 8868, 14648, null, null, null],
49+
algorithms: ['Cleora', 'Cleora (whiten)', 'RandNE', 'ProNE', 'DeepWalk', 'NetMF'],
50+
facebook: [3.9, 25.2, 39.8, 64.0, 540.8, 1047.4],
51+
ppi_large: [55.6, 335.2, 541.0, 875.8, null, null],
52+
flickr: [87.2, 524.5, 830.4, 1354.9, null, null],
53+
ogbn_arxiv: [165.4, 993.8, 1550.8, 2545.5, null, null],
54+
yelp: [700.0, null, null, null, null, null],
55+
roadnet: [1919.1, null, null, null, null, null],
5756
};
5857

5958
const SCATTER_DATA = {
6059
'ego-Facebook': {
61-
'Cleora (whiten)': { acc: 0.964, time: 0.740 },
62-
'NetMF': { acc: 0.944, time: 17.920 },
63-
'Node2Vec': { acc: 0.918, time: 111.426 },
64-
'DeepWalk': { acc: 0.912, time: 32.352 },
65-
'RandNE': { acc: 0.318, time: 0.232 },
66-
'ProNE': { acc: 0.021, time: 1.429 },
60+
'Cleora (whiten)': { acc: 0.881, time: 0.223 },
61+
'NetMF': { acc: 0.889, time: 35.229 },
62+
'DeepWalk': { acc: 0.885, time: 50.093 },
63+
'RandNE': { acc: 0.120, time: 0.070 },
64+
'ProNE': { acc: 0.019, time: 0.264 },
65+
'Cleora': { acc: 0.350, time: 0.111 },
66+
},
67+
'PPI-large': {
68+
'Cleora (whiten)': { acc: 0.985, time: 1.702 },
69+
'Cleora': { acc: 0.025, time: 0.707 },
70+
'ProNE': { acc: 0.008, time: 7.286 },
71+
'RandNE': { acc: 0.014, time: 1.863 },
72+
},
73+
'Flickr': {
74+
'Cleora (whiten)': { acc: 0.502, time: 2.218 },
75+
'Cleora': { acc: 0.157, time: 0.869 },
76+
'ProNE': { acc: 0.142, time: 10.732 },
77+
'RandNE': { acc: 0.153, time: 2.169 },
6778
},
6879
'ogbn-arxiv': {
69-
'Cleora': { acc: 0.038, time: 0.747 },
70-
'RandNE': { acc: 0.030, time: 2.046 },
71-
'ProNE': { acc: 0.026, time: 8.333 },
80+
'Cleora (whiten)': { acc: 0.624, time: 3.623 },
81+
'Cleora': { acc: 0.038, time: 1.290 },
82+
'RandNE': { acc: 0.032, time: 3.204 },
83+
'ProNE': { acc: 0.026, time: 15.725 },
7284
},
7385
};
7486

7587
const CV_DATA = {
76-
datasets: ['ego-Facebook'],
77-
meanAccuracy: [0.888],
78-
stdAccuracy: [0.015],
79-
meanF1: [0.757],
80-
stdF1: [0.021],
88+
datasets: ['ego-Facebook', 'PPI-large', 'Flickr', 'ogbn-arxiv'],
89+
meanAccuracy: [0.886, 0.985, 0.507, 0.620],
90+
stdAccuracy: [0.017, 0.001, 0.006, 0.003],
91+
meanF1: [0.710, 0.985, 0.507, 0.620],
92+
stdF1: [0.019, 0.001, 0.006, 0.003],
8193
};
8294

8395
function chartDefaults() {
@@ -116,7 +128,7 @@ function buildAccuracyChart() {
116128
maintainAspectRatio: false,
117129
interaction: { mode: 'index', intersect: false },
118130
plugins: {
119-
title: { display: true, text: 'Accuracy per Algorithm — All Datasets', color: COLORS.text, font: { size: 16, weight: 500 }, padding: { bottom: 20 } },
131+
title: { display: true, text: 'Node Classification Accuracy — 256 dim, All Datasets', color: COLORS.text, font: { size: 16, weight: 500 }, padding: { bottom: 20 } },
120132
tooltip: {
121133
callbacks: {
122134
label: ctx => {
@@ -163,14 +175,13 @@ function buildSpeedChart() {
163175
responsive: true,
164176
maintainAspectRatio: false,
165177
plugins: {
166-
title: { display: true, text: 'Embedding Time (seconds, log scale)', color: COLORS.text, font: { size: 16, weight: 500 }, padding: { bottom: 20 } },
178+
title: { display: true, text: 'Embedding Time (seconds) — Linear Scale', color: COLORS.text, font: { size: 16, weight: 500 }, padding: { bottom: 20 } },
167179
tooltip: {
168180
callbacks: { label: ctx => ctx.raw !== null ? ctx.dataset.label + ': ' + ctx.raw + 's' : '' }
169181
},
170182
},
171183
scales: {
172184
x: {
173-
type: 'logarithmic',
174185
grid: { color: COLORS.border + '60' },
175186
ticks: { color: COLORS.textMuted, callback: v => v + 's' },
176187
},
@@ -208,15 +219,14 @@ function buildMemoryChart() {
208219
responsive: true,
209220
maintainAspectRatio: false,
210221
plugins: {
211-
title: { display: true, text: 'Peak Memory Usage (MB, log scale)', color: COLORS.text, font: { size: 16, weight: 500 }, padding: { bottom: 20 } },
222+
title: { display: true, text: 'Peak Memory Usage (MB) — Linear Scale', color: COLORS.text, font: { size: 16, weight: 500 }, padding: { bottom: 20 } },
212223
tooltip: {
213224
callbacks: { label: ctx => ctx.raw !== null ? ctx.dataset.label + ': ' + ctx.raw + ' MB' : '' }
214225
},
215226
},
216227
scales: {
217228
x: { grid: { display: false }, ticks: { color: COLORS.textMuted } },
218229
y: {
219-
type: 'logarithmic',
220230
grid: { color: COLORS.border + '60' },
221231
ticks: { color: COLORS.textMuted, callback: v => v + ' MB' },
222232
},
@@ -229,6 +239,8 @@ function buildScatterChart() {
229239
const ctx = document.getElementById('chart-scatter').getContext('2d');
230240
const datasetColors = {
231241
'ego-Facebook': COLORS.accent,
242+
'PPI-large': '#10b981',
243+
'Flickr': COLORS.orange,
232244
'ogbn-arxiv': COLORS.blue,
233245
};
234246

@@ -253,7 +265,7 @@ function buildScatterChart() {
253265
responsive: true,
254266
maintainAspectRatio: false,
255267
plugins: {
256-
title: { display: true, text: 'Accuracy vs Embedding Time', color: COLORS.text, font: { size: 16, weight: 500 }, padding: { bottom: 20 } },
268+
title: { display: true, text: 'Accuracy vs Embedding Time — 256 dim', color: COLORS.text, font: { size: 16, weight: 500 }, padding: { bottom: 20 } },
257269
tooltip: {
258270
callbacks: {
259271
label: ctx => {
@@ -265,7 +277,6 @@ function buildScatterChart() {
265277
},
266278
scales: {
267279
x: {
268-
type: 'logarithmic',
269280
title: { display: true, text: 'Embedding Time (seconds)', color: COLORS.textMuted },
270281
grid: { color: COLORS.border + '60' },
271282
ticks: { color: COLORS.textMuted, callback: v => v + 's' },
@@ -358,7 +369,7 @@ function buildCVChart() {
358369
responsive: true,
359370
maintainAspectRatio: false,
360371
plugins: {
361-
title: { display: true, text: 'Cross-Validation: Cleora on ego-Facebook (1024 dim, 4 iter)', color: COLORS.text, font: { size: 16, weight: 500 }, padding: { bottom: 20 } },
372+
title: { display: true, text: 'Cross-Validation: Cleora (whiten, 8 iter, 256 dim)', color: COLORS.text, font: { size: 16, weight: 500 }, padding: { bottom: 20 } },
362373
tooltip: {
363374
callbacks: {
364375
label: ctx => {

0 commit comments

Comments
 (0)