@@ -98,61 +98,72 @@ class BehaviorVideoJob(GenericEtl[BehaviorVideoJobSettings]):
9898 run_job() -> JobResponse
9999 """
100100
101- def _run_compression (
101+ def _run_parallel (
102102 self ,
103103 convert_video_args : list [tuple [Path , Path , tuple [str , str ] | None ]],
104- ) -> None :
105- """
106- Runs CompressionRequests at the specified paths.
107- """
104+ ) -> list [tuple [Path , CalledProcessError ]]:
105+ """Run conversions in a ProcessPoolExecutor, collecting failures."""
108106 errors : list [tuple [Path , CalledProcessError ]] = []
109- if self .job_settings .parallel_compression :
110- # Execute in-parallel
111- if len (convert_video_args ) == 0 :
112- return
113-
114- num_jobs = len (convert_video_args )
115- with ProcessPoolExecutor (max_workers = num_jobs ) as executor :
116- futures = {
117- executor .submit (
118- convert_video ,
119- * params ,
120- self .job_settings .ffmpeg_thread_cnt ,
121- ): params
122- for params in convert_video_args
123- }
124- for future in as_completed (futures ):
125- video_path = futures [future ][0 ]
126- try :
127- result = future .result ()
128- except CalledProcessError as exc :
129- errors .append ((video_path , exc ))
130- else :
131- logger .info ("FFmpeg job completed: %s" , result )
132- else :
133- # Execute serially
134- for params in convert_video_args :
135- video_path = params [0 ]
107+ if not convert_video_args :
108+ return errors
109+ thread_cnt = self .job_settings .ffmpeg_thread_cnt
110+ with ProcessPoolExecutor (max_workers = len (convert_video_args )) as ex :
111+ futures = {
112+ ex .submit (convert_video , * params , thread_cnt ): params
113+ for params in convert_video_args
114+ }
115+ for future in as_completed (futures ):
116+ video_path = futures [future ][0 ]
136117 try :
137- result = convert_video (
138- * params , self .job_settings .ffmpeg_thread_cnt
139- )
118+ result = future .result ()
140119 except CalledProcessError as exc :
141120 errors .append ((video_path , exc ))
142121 else :
143122 logger .info ("FFmpeg job completed: %s" , result )
123+ return errors
124+
125+ def _run_serial (
126+ self ,
127+ convert_video_args : list [tuple [Path , Path , tuple [str , str ] | None ]],
128+ ) -> list [tuple [Path , CalledProcessError ]]:
129+ """Run conversions one at a time, collecting failures."""
130+ errors : list [tuple [Path , CalledProcessError ]] = []
131+ thread_cnt = self .job_settings .ffmpeg_thread_cnt
132+ for params in convert_video_args :
133+ video_path = params [0 ]
134+ try :
135+ result = convert_video (* params , thread_cnt )
136+ except CalledProcessError as exc :
137+ errors .append ((video_path , exc ))
138+ else :
139+ logger .info ("FFmpeg job completed: %s" , result )
140+ return errors
144141
145- if errors :
146- formatted = [
147- _format_ffmpeg_error (video_path , exc )
148- for video_path , exc in errors
149- ]
150- for block in formatted :
151- logger .error (block )
152- raise RuntimeError (
153- f"{ len (errors )} ffmpeg job(s) failed:\n \n "
154- + "\n \n " .join (formatted )
155- )
142+ def _run_compression (
143+ self ,
144+ convert_video_args : list [tuple [Path , Path , tuple [str , str ] | None ]],
145+ ) -> None :
146+ """
147+ Runs CompressionRequests at the specified paths.
148+ """
149+ if self .job_settings .parallel_compression :
150+ errors = self ._run_parallel (convert_video_args )
151+ else :
152+ errors = self ._run_serial (convert_video_args )
153+
154+ if not errors :
155+ return
156+
157+ formatted = [
158+ _format_ffmpeg_error (video_path , exc )
159+ for video_path , exc in errors
160+ ]
161+ for block in formatted :
162+ logger .error (block )
163+ raise RuntimeError (
164+ f"{ len (errors )} ffmpeg job(s) failed:\n \n "
165+ + "\n \n " .join (formatted )
166+ )
156167
157168 def run_job (self ) -> JobResponse :
158169 """
0 commit comments