add a simple script for plotting public vs secret run times

Author: ngc92

pyproject.toml

@@ -23,6 +23,8 @@ dependencies = [
     "fastapi[all]",
     "uvicorn",
     "jinja2",
+    "matplotlib>=3.10.8",
+    "numpy>=2.2.6",
 ]
 
 [project.optional-dependencies]

src/kernelbot/cogs/admin_cog.py

@@ -1,3 +1,4 @@
+import io
 import json
 import subprocess
 import tempfile
@@ -102,6 +103,10 @@ def __init__(self, bot: "ClusterBot"):
             name="show-stats", description="Show stats for the kernelbot"
         )(self.show_bot_stats)
 
+        self.show_timing_correlation = bot.admin_group.command(
+            name="show-timing-correlation", description="Show correlation between public and private run timings"
+        )(self.show_timing_correlation)
+
         self.resync = bot.admin_group.command(
             name="resync", description="Trigger re-synchronization of slash commands"
         )(self.resync)
@@ -793,6 +798,67 @@ async def show_bot_stats(self, interaction: discord.Interaction, last_day_only:
             msg += "\n```"
             await send_discord_message(interaction, msg, ephemeral=True)
 
+    @with_error_handling
+    @discord.app_commands.describe(leaderboard="Which leaderboard")
+    @discord.app_commands.autocomplete(leaderboard=leaderboard_name_autocomplete)
+    async def show_timing_correlation(self, interaction: discord.Interaction, leaderboard: str):
+        is_admin = await self.admin_check(interaction)
+        if not is_admin:
+            await send_discord_message(
+                interaction,
+                "You need to have Admin permissions to run this command",
+                ephemeral=True,
+            )
+            return
+
+        with self.bot.leaderboard_db as db:
+            stats = db.generate_correlation_stats(leaderboard)
+
+            if len(stats) == 0:
+                await send_discord_message(
+                    interaction,
+                    f"No ranked runs (with matching public/secret scores) found for leaderboard {leaderboard}.")
+                return
+
+            import matplotlib.pyplot as plt
+            import numpy as np
+
+            files = []
+            for runner, pairs in stats.items():
+                public_scores, secret_scores = zip(*pairs)
+
+                plt.figure(figsize=(8, 6))
+                plt.scatter(public_scores, secret_scores, alpha=0.6)
+
+                # Diagonal reference line
+                min_val = min(min(public_scores), min(secret_scores))
+                max_val = max(max(public_scores), max(secret_scores))
+                plt.plot([min_val, max_val], [min_val, max_val], 'r--', alpha=0.5, label='y=x')
+
+                # Calculate correlation
+                corr = np.corrcoef(public_scores, secret_scores)[0, 1]
+
+                plt.xlabel('Public Score')
+                plt.ylabel('Secret Score')
+                plt.title(f'{leaderboard} on {runner} - Correlation: {corr:.3f}')
+                plt.legend()
+                plt.grid(True, alpha=0.3)
+
+                plt.tight_layout()
+
+                buf = io.BytesIO()
+                plt.savefig(buf, format='png', dpi=150)
+                buf.seek(0)
+
+                files.append(discord.File(buf, filename=f'correlation_{runner}.png'))
+                plt.close()
+
+            await send_discord_message(
+                interaction,
+                "Correlation of running times for public and private runs",
+                files=files,
+                ephemeral=True)
+
     @with_error_handling
     async def resync(self, interaction: discord.Interaction):
         """Admin command to resync slash commands"""

src/libkernelbot/leaderboard_db.py

@@ -1,6 +1,7 @@
 import dataclasses
 import datetime
 import json
+from collections import defaultdict
 from typing import Dict, List, Optional
 
 import psycopg2
@@ -812,6 +813,50 @@ def _generate_stats(self, last_day: bool = False):
 
         return result
 
+    def generate_correlation_stats(self, leaderboard_name: str) -> dict[str, list]:
+        query = """
+        SELECT
+            s.id,
+            r.score,
+            r.runner,
+            r.secret
+        FROM leaderboard.runs r
+        JOIN leaderboard.submission s ON r.submission_id = s.id
+        JOIN leaderboard.leaderboard l ON s.leaderboard_id = l.id
+        WHERE
+            l.name = %s
+            AND r.score IS NOT NULL
+            AND r.passed
+        """
+
+        self.cursor.execute(query, (leaderboard_name,))
+
+        matches = defaultdict(list)
+        for run in self.cursor.fetchall():
+            id, score, runner, secret = run
+            matches[(id, runner)].append((float(score), secret))
+
+        results = defaultdict(list)
+        for entry, values in matches.items():
+            if len(values) != 2:
+                continue
+            secret = None
+            public = None
+            if values[0][1]:
+                secret = values[0][0]
+            else:
+                public = values[0][0]
+
+            if values[1][1]:
+                secret = values[1][0]
+            else:
+                public = values[1][0]
+
+            if secret is not None and public is not None:
+                results[entry[1]].append((public, secret))
+
+        return results
+
     def get_user_from_id(self, id: str) -> Optional[str]:
         try:
             self.cursor.execute(