Skip to content

Commit 941f5a8

Browse files
msaroufimmsaroufim
authored
Hide submissions with failed secret runs (#490)
1 parent 8f9d13f commit 941f5a8

2 files changed

Lines changed: 118 additions & 2 deletions

File tree

‎src/libkernelbot/leaderboard_db.py‎

Lines changed: 49 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -884,6 +884,15 @@ def get_leaderboard_submissions(
884884
AND r.score IS NOT NULL
885885
AND r.passed
886886
AND s.user_id = %s
887+
AND NOT EXISTS (
888+
SELECT 1
889+
FROM leaderboard.runs sr
890+
WHERE sr.submission_id = s.id
891+
AND sr.secret
892+
AND sr.runner = r.runner
893+
AND sr.mode = r.mode
894+
AND sr.passed = FALSE
895+
)
887896
ORDER BY r.score ASC
888897
LIMIT %s OFFSET %s
889898
"""
@@ -905,6 +914,15 @@ def get_leaderboard_submissions(
905914
JOIN leaderboard.user_info ui ON s.user_id = ui.id
906915
WHERE l.name = %s AND r.runner = %s AND NOT r.secret
907916
AND r.score IS NOT NULL AND r.passed
917+
AND NOT EXISTS (
918+
SELECT 1
919+
FROM leaderboard.runs sr
920+
WHERE sr.submission_id = s.id
921+
AND sr.secret
922+
AND sr.runner = r.runner
923+
AND sr.mode = r.mode
924+
AND sr.passed = FALSE
925+
)
908926
ORDER BY s.user_id, r.score ASC
909927
)
910928
SELECT
@@ -1244,8 +1262,19 @@ def get_user_submissions(
12441262
submission_ids = [row[0] for row in submissions]
12451263
runs_query = """
12461264
SELECT submission_id, runner as gpu_type, score
1247-
FROM leaderboard.runs
1248-
WHERE submission_id = ANY(%s) AND NOT secret AND passed
1265+
FROM leaderboard.runs r
1266+
WHERE submission_id = ANY(%s)
1267+
AND NOT secret
1268+
AND passed
1269+
AND NOT EXISTS (
1270+
SELECT 1
1271+
FROM leaderboard.runs sr
1272+
WHERE sr.submission_id = r.submission_id
1273+
AND sr.secret
1274+
AND sr.runner = r.runner
1275+
AND sr.mode = r.mode
1276+
AND sr.passed = FALSE
1277+
)
12491278
"""
12501279
self.cursor.execute(runs_query, (submission_ids,))
12511280
runs_by_submission: dict = {}
@@ -1384,6 +1413,15 @@ def get_leaderboard_submission_count(
13841413
AND r.score IS NOT NULL
13851414
AND r.passed
13861415
AND s.user_id = %s
1416+
AND NOT EXISTS (
1417+
SELECT 1
1418+
FROM leaderboard.runs sr
1419+
WHERE sr.submission_id = s.id
1420+
AND sr.secret
1421+
AND sr.runner = r.runner
1422+
AND sr.mode = r.mode
1423+
AND sr.passed = FALSE
1424+
)
13871425
"""
13881426
args = (leaderboard_name, gpu_name, user_id)
13891427
else:
@@ -1397,6 +1435,15 @@ def get_leaderboard_submission_count(
13971435
AND NOT r.secret
13981436
AND r.score IS NOT NULL
13991437
AND r.passed
1438+
AND NOT EXISTS (
1439+
SELECT 1
1440+
FROM leaderboard.runs sr
1441+
WHERE sr.submission_id = s.id
1442+
AND sr.secret
1443+
AND sr.runner = r.runner
1444+
AND sr.mode = r.mode
1445+
AND sr.passed = FALSE
1446+
)
14001447
"""
14011448
args = (leaderboard_name, gpu_name)
14021449

‎tests/test_leaderboard_db.py‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -368,6 +368,75 @@ def test_leaderboard_submission_ranked(database, submit_leaderboard):
368368
},
369369
]
370370

371+
372+
def test_failed_secret_run_hides_submission_from_rankings(database, submit_leaderboard):
373+
submit_time = datetime.datetime.now(tz=datetime.timezone.utc)
374+
failed_secret = dataclasses.replace(sample_run_result(), passed=False)
375+
376+
with database as db:
377+
hacked = db.create_submission(
378+
"submit-leaderboard", "fast.py", 5, "fast", submit_time, user_name="user5"
379+
)
380+
_create_submission_run(db, hacked, mode="leaderboard", runner="A100", score=1.0)
381+
_create_submission_run(
382+
db,
383+
hacked,
384+
mode="leaderboard",
385+
secret=True,
386+
runner="A100",
387+
score=None,
388+
result=failed_secret,
389+
)
390+
db.mark_submission_done(hacked)
391+
392+
valid = db.create_submission(
393+
"submit-leaderboard", "valid.py", 6, "valid", submit_time, user_name="user6"
394+
)
395+
_create_submission_run(db, valid, mode="leaderboard", runner="A100", score=2.0)
396+
_create_submission_run(
397+
db,
398+
valid,
399+
mode="leaderboard",
400+
secret=True,
401+
runner="A100",
402+
score=None,
403+
)
404+
db.mark_submission_done(valid)
405+
406+
with database as db:
407+
ranked = db.get_leaderboard_submissions("submit-leaderboard", "A100")
408+
assert [row["submission_id"] for row in ranked] == [valid]
409+
assert db.get_leaderboard_submission_count("submit-leaderboard", "A100") == 1
410+
assert db.get_leaderboard_submission_count("submit-leaderboard", "A100", "5") == 0
411+
412+
413+
def test_failed_secret_run_hides_user_submission_scores(database, submit_leaderboard):
414+
submit_time = datetime.datetime.now(tz=datetime.timezone.utc)
415+
failed_secret = dataclasses.replace(sample_run_result(), passed=False)
416+
417+
with database as db:
418+
sub_id = db.create_submission(
419+
"submit-leaderboard", "fast.py", 5, "fast", submit_time, user_name="user5"
420+
)
421+
_create_submission_run(db, sub_id, mode="leaderboard", runner="A100", score=1.0)
422+
_create_submission_run(
423+
db,
424+
sub_id,
425+
mode="leaderboard",
426+
secret=True,
427+
runner="A100",
428+
score=None,
429+
result=failed_secret,
430+
)
431+
db.mark_submission_done(sub_id)
432+
433+
with database as db:
434+
submissions = db.get_user_submissions("5", leaderboard_name="submit-leaderboard")
435+
assert len(submissions) == 1
436+
assert submissions[0]["id"] == sub_id
437+
assert submissions[0]["runs"] == []
438+
439+
371440
def test_validate_identity_web_auth_happy_path(database, submit_leaderboard):
372441
with database as db:
373442
db.cursor.execute(

0 commit comments

Comments
 (0)