Merge pull request #7035 from BOINC/dpa_cpu_sched

client: fix sched bug that could cause idle devices

Author: AenBleidd

client/cpu_sched.cpp

@@ -23,17 +23,12 @@
 //          - for a given resource, jobs in deadline danger first
 //          - jobs from projects with lower recent est. credit first
 //      In principle, the run list could include all runnable jobs.
-//      For efficiency, we stop adding:
+//      But for efficiency, we stop adding:
 //          - GPU jobs: when all GPU instances used
-//          - CPU jobs: when the # of CPUs allocated to single-thread jobs,
-//              OR the # allocated to multi-thread jobs, exceeds # CPUs
-//              (ensure we have enough single-thread jobs
-//              in case we can't run the multi-thread jobs)
-//      NOTE: RAM usage is not taken into consideration
-//          in the process of building this list.
-//          It's possible that we include a bunch of jobs that can't run
-//          because of memory limits,
-//          even though there are other jobs that could run.
+//          - CPU jobs: when all CPUs used, with a 2X slop factor
+//              so we have enough jobs even if some can't run
+//              because of their RAM usage.
+//      (RAM usage is not taken into consideration in building this list)
 //      - add running jobs to the list
 //          (in case they haven't finished time slice or checkpointed)
 //      - sort the list according to "more_important()"
@@ -43,9 +38,7 @@
 //      other jobs (enforce_run_list).
 //      Don't run a job if
 //      - its GPUs can't be assigned (possible if need >1 GPU)
-//      - it's a multi-thread job, and CPU usage would be #CPUs+1 or more
-//      - it's a single-thread job, don't oversaturate CPU
-//          (details depend on whether a MT job is running)
+//      - it would use too many CPUs (see details below)
 //      - its memory usage would exceed RAM limits
 //          If there's a running job using a given app version,
 //          unstarted jobs using that app version
@@ -103,28 +96,42 @@ struct PROC_RESOURCES {
         pr_coprocs.clear_usage();
     }
 
-    // should we stop scanning jobs?
+    // In make_run_list(), should we stop scanning CPU jobs?
+    // factors:
+    // - we need to stop sometime; if there are 1000 jobs
+    //   it would be inefficient to scan them all
+    // - some jobs might use too much RAM to run,
+    //   so we scan jobs up to 2X #CPUs
     //
     inline bool stop_scan_cpu() {
-        if (ncpus_used_st >= ncpus) return true;
+        if (ncpus_used_st >= 2*ncpus) {
+            return true;
+        }
         if (ncpus_used_mt >= 2*ncpus) return true;
             // kind of arbitrary, but need to have some limit
             // in case there are only MT jobs, and lots of them
         return false;
     }
 
+    // similar, for GPU
+    //
     inline bool stop_scan_coproc(int rsc_type) {
         COPROC& cp = pr_coprocs.coprocs[rsc_type];
         for (int i=0; i<cp.count; i++) {
             if (cp.usage[i] < 1) return false;
+                // could make it 2 to provide more jobs
+                // for RAM exceeded case
         }
         return true;
     }
 
-    // Should we add this job to the runnable list?
-    // There are two possible reasons not to:
+    // Should we add this job to the run list?
+    // reasons not to:
     // - the job won't be able to run (e.g. it's being aborted)
-    // - the runnable list has enough jobs of this type already
+    // - it's backed off
+    // - it's GPU, and GPUs are suspended
+    // - it's Docker, and Docker not installed or not inited
+    // - it's GPU, and run list already commits the needed GPUs
     //
     bool can_schedule(RESULT* rp, ACTIVE_TASK* atp) {
         if (atp) {
@@ -157,20 +164,6 @@ struct PROC_RESOURCES {
             if (!sufficient_coprocs(*rp)) {
                 return false;
             }
-        } else {
-            // CPU jobs: see if we have enough already in list
-            //
-            if (rp->resource_usage.avg_ncpus > 1) {
-                if (ncpus_used_mt > 0) {
-                    if (ncpus_used_mt + rp->resource_usage.avg_ncpus > ncpus) {
-                        return false;
-                    }
-                }
-            } else {
-                if (ncpus_used_st >= ncpus) {
-                    return false;
-                }
-            }
         }
 
         // if job uses Docker, make sure it's installed
@@ -252,6 +245,8 @@ struct PROC_RESOURCES {
         adjust_rec_sched(rp);
     }
 
+    // are there enough unused coproc instances to run this job?
+    //
     bool sufficient_coprocs(RESULT& r) {
         int rt = r.resource_usage.rsc_type;
         if (!rt) return true;
@@ -358,8 +353,7 @@ static inline bool finished_time_slice(ACTIVE_TASK* atp) {
 // Values are returned in project->next_runnable_result
 // (skip projects for which this is already non-NULL)
 //
-// Don't choose results with already_selected == true;
-// mark chosen results as already_selected.
+// Don't choose results with already_selected == true
 //
 // The preference order:
 // 1. results with active tasks that are running
@@ -414,15 +408,6 @@ void CLIENT_STATE::assign_results_to_projects() {
         if (project->next_runnable_result) continue;
         project->next_runnable_result = rp;
     }
-
-    // mark selected results, so CPU scheduler won't try to consider
-    // a result more than once
-    //
-    for (PROJECT *project: projects) {
-        if (project->next_runnable_result) {
-            project->next_runnable_result->already_selected = true;
-        }
-    }
 }
 
 // Among projects with a "next runnable result",
@@ -435,14 +420,18 @@ RESULT* CLIENT_STATE::highest_prio_project_best_result() {
     bool first = true;
 
     for (PROJECT *p: projects) {
-        if (!p->next_runnable_result) continue;
+        if (!p->next_runnable_result) {
+            continue;
+        }
         if (first || p->sched_priority > best_prio) {
             first = false;
             best_project = p;
             best_prio = p->sched_priority;
         }
     }
-    if (!best_project) return NULL;
+    if (!best_project) {
+        return NULL;
+    }
 
     RESULT* rp = best_project->next_runnable_result;
     best_project->next_runnable_result = 0;
@@ -892,7 +881,7 @@ void CLIENT_STATE::make_run_list(vector<RESULT*>& run_list) {
     PROC_RESOURCES proc_rsc;
 
     if (log_flags.cpu_sched_debug) {
-        msg_printf(0, MSG_INFO, "[cpu_sched_debug] schedule_cpus(): start");
+        msg_printf(0, MSG_INFO, "[cpu_sched_debug] make_run_list(): start");
     }
 
     proc_rsc.init();
@@ -980,13 +969,17 @@ void CLIENT_STATE::make_run_list(vector<RESULT*>& run_list) {
 #endif
     while (!proc_rsc.stop_scan_cpu()) {
         RESULT *rp = earliest_deadline_result(RSC_TYPE_CPU);
-        if (!rp) break;
+        if (!rp) {
+            break;
+        }
         rp->already_selected = true;
         if (have_max_concurrent && max_concurrent_exceeded(rp)) {
             continue;
         }
         ACTIVE_TASK *atp = lookup_active_task_by_result(rp);
-        if (!proc_rsc.can_schedule(rp, atp)) continue;
+        if (!proc_rsc.can_schedule(rp, atp)) {
+            continue;
+        }
         proc_rsc.schedule(rp, atp, true);
         rp->project->rsc_pwf[0].deadlines_missed_copy--;
         rp->edf_scheduled = true;
@@ -1010,17 +1003,24 @@ void CLIENT_STATE::make_run_list(vector<RESULT*>& run_list) {
         if (!rp) {
             break;
         }
+        rp->already_selected = true;
         if (have_max_concurrent && max_concurrent_exceeded(rp)) {
             continue;
         }
         ACTIVE_TASK *atp = lookup_active_task_by_result(rp);
-        if (!proc_rsc.can_schedule(rp, atp)) continue;
+        if (!proc_rsc.can_schedule(rp, atp)) {
+            continue;
+        }
         proc_rsc.schedule(rp, atp, false);
         run_list.push_back(rp);
         if (have_max_concurrent) {
             max_concurrent_inc(rp);
         }
     }
+
+    if (log_flags.cpu_sched_debug) {
+        msg_printf(0, MSG_INFO, "[cpu_sched_debug] make_run_list(): end");
+    }
 }
 
 static inline bool in_run_list(vector<RESULT*>& run_list, ACTIVE_TASK* atp) {

client/sim.cpp

@@ -1445,6 +1445,8 @@ void do_client_simulation() {
     set_no_rsc_config();
     process_gpu_exclusions();
 
+    gstate.init_result_resource_usage();
+
     get_app_params();
     if (!include_empty_projects) {
         cull_projects();