Skip to content

Commit 62778e3

Browse files
RishiDesaiRishiDesai
committed
delete exp now stop analysis jobs
1 parent 8276e80 commit 62778e3

3 files changed

Lines changed: 147 additions & 27 deletions

File tree

‎frontend/src/components/experiment-trials-table.tsx‎

Lines changed: 37 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,39 @@ const PassAtOneLeaderboard = dynamic(
8888
},
8989
);
9090

91+
const ACTIVE_TRIAL_STATUSES = [
92+
"running",
93+
"queued",
94+
"retrying",
95+
"pending",
96+
] as const;
97+
const ACTIVE_PIPELINE_STATUSES = ["pending", "queued", "running"] as const;
98+
99+
function isActiveTrialStatus(status: string | null | undefined): boolean {
100+
return ACTIVE_TRIAL_STATUSES.includes(
101+
status as (typeof ACTIVE_TRIAL_STATUSES)[number],
102+
);
103+
}
104+
105+
function isActivePipelineStatus(status: string | null | undefined): boolean {
106+
return ACTIVE_PIPELINE_STATUSES.includes(
107+
status as (typeof ACTIVE_PIPELINE_STATUSES)[number],
108+
);
109+
}
110+
111+
function taskHasCancellableWork(task: Task): boolean {
112+
return (
113+
task.status === "analyzing" ||
114+
task.status === "verdict_pending" ||
115+
isActivePipelineStatus(task.verdict_status) ||
116+
(task.trials ?? []).some(
117+
(trial) =>
118+
isActiveTrialStatus(trial.status) ||
119+
isActivePipelineStatus(trial.analysis_status),
120+
)
121+
);
122+
}
123+
91124
export type AgentSummary = ExperimentAgentSummary;
92125

93126
type ExperimentTrialsTableProps = {
@@ -847,12 +880,7 @@ export function ExperimentTrialsTable({
847880
}, [selectedTaskList]);
848881

849882
const selectedCancellableTasks = useMemo(
850-
() =>
851-
selectedTaskList.filter((task) =>
852-
(task.trials ?? []).some((trial) =>
853-
["running", "queued", "retrying", "pending"].includes(trial.status),
854-
),
855-
),
883+
() => selectedTaskList.filter((task) => taskHasCancellableWork(task)),
856884
[selectedTaskList],
857885
);
858886

@@ -865,13 +893,9 @@ export function ExperimentTrialsTable({
865893
(trial) => trial.status === "failed" || trial.status === "success",
866894
);
867895
const hasAnalysisInFlight = trials.some((trial) =>
868-
["pending", "queued", "running"].includes(
869-
trial.analysis_status ?? "",
870-
),
871-
);
872-
const verdictInFlight = ["pending", "queued", "running"].includes(
873-
task.verdict_status ?? "",
896+
isActivePipelineStatus(trial.analysis_status),
874897
);
898+
const verdictInFlight = isActivePipelineStatus(task.verdict_status);
875899
return allTrialsTerminal && !hasAnalysisInFlight && !verdictInFlight;
876900
}),
877901
[selectedTaskList],
@@ -890,9 +914,7 @@ export function ExperimentTrialsTable({
890914
trial.analysis_status === "success" ||
891915
trial.analysis_status === "failed",
892916
);
893-
const verdictInFlight = ["pending", "queued", "running"].includes(
894-
task.verdict_status ?? "",
895-
);
917+
const verdictInFlight = isActivePipelineStatus(task.verdict_status);
896918
return allTrialsTerminal && allAnalysesComplete && !verdictInFlight;
897919
}),
898920
[selectedTaskList],

‎frontend/src/components/task-files-panel.tsx‎

Lines changed: 45 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,39 @@ function getNodeName(path: string): string {
101101

102102
// Truncate files larger than 100KB initially
103103
const TRUNCATE_THRESHOLD = 100 * 1024;
104+
const ACTIVE_TRIAL_STATUSES = [
105+
"running",
106+
"queued",
107+
"retrying",
108+
"pending",
109+
] as const;
110+
const ACTIVE_PIPELINE_STATUSES = ["pending", "queued", "running"] as const;
111+
112+
function isActiveTrialStatus(status: string | null | undefined): boolean {
113+
return ACTIVE_TRIAL_STATUSES.includes(
114+
status as (typeof ACTIVE_TRIAL_STATUSES)[number],
115+
);
116+
}
117+
118+
function isActivePipelineStatus(status: string | null | undefined): boolean {
119+
return ACTIVE_PIPELINE_STATUSES.includes(
120+
status as (typeof ACTIVE_PIPELINE_STATUSES)[number],
121+
);
122+
}
123+
124+
function taskHasCancellableWork(task: Task | null | undefined): boolean {
125+
if (!task) return false;
126+
return (
127+
task.status === "analyzing" ||
128+
task.status === "verdict_pending" ||
129+
isActivePipelineStatus(task.verdict_status) ||
130+
(task.trials ?? []).some(
131+
(trial) =>
132+
isActiveTrialStatus(trial.status) ||
133+
isActivePipelineStatus(trial.analysis_status),
134+
)
135+
);
136+
}
104137

105138
function formatFileSize(bytes: number): string {
106139
if (bytes < 1024) return `${bytes} B`;
@@ -412,18 +445,23 @@ export function TaskFilesPanel({
412445
const canRetryTask = allowRetry && retryableTrials.length > 0;
413446
const activeTrials = useMemo(() => {
414447
if (!task?.trials) return [];
415-
return task.trials.filter((trial) =>
416-
["running", "queued", "retrying", "pending"].includes(trial.status),
417-
);
448+
return task.trials.filter((trial) => isActiveTrialStatus(trial.status));
418449
}, [task]);
419-
const canCancelTask = allowRetry && activeTrials.length > 0;
450+
const canCancelTask = allowRetry && taskHasCancellableWork(task);
451+
const cancelActionLabel =
452+
activeTrials.length > 0
453+
? `Cancel (${activeTrials.length})`
454+
: task?.status === "verdict_pending" ||
455+
isActivePipelineStatus(task?.verdict_status)
456+
? "Cancel verdict"
457+
: "Cancel analysis";
420458
const allTrialsTerminal =
421459
Boolean(task?.trials?.length) &&
422460
(task?.trials ?? []).every(
423461
(trial) => trial.status === "failed" || trial.status === "success",
424462
);
425463
const hasAnalysisInFlight = (task?.trials ?? []).some((trial) =>
426-
["pending", "queued", "running"].includes(trial.analysis_status ?? ""),
464+
isActivePipelineStatus(trial.analysis_status),
427465
);
428466
const allAnalysesComplete =
429467
Boolean(task?.trials?.length) &&
@@ -432,9 +470,7 @@ export function TaskFilesPanel({
432470
trial.analysis_status === "success" ||
433471
trial.analysis_status === "failed",
434472
);
435-
const verdictInFlight = ["pending", "queued", "running"].includes(
436-
verdictSource?.verdict_status ?? "",
437-
);
473+
const verdictInFlight = isActivePipelineStatus(verdictSource?.verdict_status);
438474
const canRunTaskAnalysis =
439475
allowRetry &&
440476
Boolean(task) &&
@@ -1356,9 +1392,7 @@ export function TaskFilesPanel({
13561392
) : (
13571393
<OctagonX className="mr-1 h-3.5 w-3.5" />
13581394
)}
1359-
{isCancelling
1360-
? "Cancelling..."
1361-
: `Cancel (${activeTrials.length})`}
1395+
{isCancelling ? "Cancelling..." : cancelActionLabel}
13621396
</Button>
13631397
)}
13641398
{allowRetry && (

‎oddish/src/oddish/core/endpoints.py‎

Lines changed: 65 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
TrialModel,
3535
TrialStatus,
3636
VerdictStatus,
37+
utcnow,
3738
)
3839
from oddish.schemas import (
3940
TaskBrowseExperiment,
@@ -758,6 +759,44 @@ def _reset_task_verdict(task: TaskModel) -> None:
758759
task.verdict_finished_at = None
759760

760761

762+
def _task_has_active_analysis(task: TaskModel) -> bool:
763+
return any(
764+
trial.analysis_status
765+
in (AnalysisStatus.PENDING, AnalysisStatus.QUEUED, AnalysisStatus.RUNNING)
766+
for trial in task.trials or []
767+
)
768+
769+
770+
def _task_has_active_trials(task: TaskModel) -> bool:
771+
return any(
772+
trial.status
773+
in (
774+
TrialStatus.PENDING,
775+
TrialStatus.QUEUED,
776+
TrialStatus.RUNNING,
777+
TrialStatus.RETRYING,
778+
)
779+
for trial in task.trials or []
780+
)
781+
782+
783+
def _clear_stale_task_pipeline_status(task: TaskModel) -> None:
784+
"""Move a surviving task out of pipeline-only states after scoped deletion."""
785+
if task.status not in (TaskStatus.ANALYZING, TaskStatus.VERDICT_PENDING):
786+
return
787+
if _task_has_active_trials(task) or _task_has_active_analysis(task):
788+
return
789+
if task.verdict_status in (
790+
VerdictStatus.PENDING,
791+
VerdictStatus.QUEUED,
792+
VerdictStatus.RUNNING,
793+
):
794+
return
795+
796+
task.status = TaskStatus.COMPLETED if task.trials else TaskStatus.FAILED
797+
task.finished_at = task.finished_at or utcnow()
798+
799+
761800
def _reset_trial_analysis(trial: TrialModel) -> None:
762801
"""Clear cached analysis state before re-running analysis."""
763802
trial.analysis = None
@@ -1311,6 +1350,25 @@ async def delete_experiment_core(
13111350
linked_task_ids = [row[0] for row in linked_task_rows]
13121351
linked_task_s3 = {row[0]: (row[1], row[2]) for row in linked_task_rows}
13131352

1353+
if linked_task_ids:
1354+
await session.execute(
1355+
text(
1356+
"""
1357+
UPDATE worker_jobs
1358+
SET status = 'CANCELLED',
1359+
finished_at = NOW(),
1360+
error_message = 'Experiment deleted by user',
1361+
current_worker_id = NULL,
1362+
current_queue_slot = NULL,
1363+
modal_function_call_id = NULL
1364+
WHERE subject_table = 'tasks'
1365+
AND subject_id = ANY(:task_ids)
1366+
AND status::text IN ('QUEUED', 'RETRYING', 'RUNNING', 'BLOCKED')
1367+
"""
1368+
),
1369+
{"task_ids": linked_task_ids},
1370+
)
1371+
13141372
# Trials scoped to this experiment.
13151373
trial_where = [TrialModel.experiment_id == experiment_id]
13161374
if org_id is not None:
@@ -1412,9 +1470,15 @@ async def delete_experiment_core(
14121470
deleted_tasks += int(task_del_result.rowcount or 0) # type: ignore[attr-defined]
14131471
task_s3_to_delete.append(linked_task_s3[tid])
14141472
else:
1415-
task = await session.get(TaskModel, tid)
1473+
task_result = await session.execute(
1474+
select(TaskModel)
1475+
.options(selectinload(TaskModel.trials))
1476+
.where(TaskModel.id == tid)
1477+
)
1478+
task = task_result.scalar_one_or_none()
14161479
if task is not None:
14171480
_reset_task_verdict(task)
1481+
_clear_stale_task_pipeline_status(task)
14181482

14191483
s3_prefixes = collect_s3_prefixes_for_deletion(
14201484
tasks=task_s3_to_delete,

0 commit comments

Comments
 (0)