Use parserParams.pExtVideoInfo instead of a BSF

Author: NicolasHug

src/torchcodec/_core/BetaCudaDeviceInterface.cpp

@@ -339,6 +339,21 @@ void BetaCudaDeviceInterface::initialize(
   parserParams.pfnDecodePicture = pfnDecodePictureCallback;
   parserParams.pfnDisplayPicture = pfnDisplayPictureCallback;
 
+  // Some containers (e.g. MP4/MOV) store codec config (H.264 SPS/PPS,
+  // MPEG-4 VOS/VOL, etc.) in extradata rather than inline in the
+  // bitstream. The NVCUVID parser needs this data to initialize, so we
+  // pass it via pExtVideoInfo. Same approach as DALI and FFmpeg cuviddec.
+  // DALI does the same thing
+  // https://github.com/NVIDIA/DALI/blob/ae79f316ae9b14c464d9cb98465f7f783da9ea89/dali/operators/video/frames_decoder_gpu.cc#L402-L408
+  if (codecPar->extradata_size > 0) {
+    auto seqhdrSize = std::min(
+        static_cast<size_t>(codecPar->extradata_size),
+        sizeof(parserExtInfo_.raw_seqhdr_data));
+    parserExtInfo_.format.seqhdr_data_length = seqhdrSize;
+    memcpy(parserExtInfo_.raw_seqhdr_data, codecPar->extradata, seqhdrSize);
+    parserParams.pExtVideoInfo = &parserExtInfo_;
+  }
+
   CUresult result = cuvidCreateVideoParser(&videoParser_, &parserParams);
   STD_TORCH_CHECK(
       result == CUDA_SUCCESS, "Failed to create video parser: ", result);
@@ -390,12 +405,6 @@ void BetaCudaDeviceInterface::initializeBSF(
           avFormatCtx->iformat->name ? avFormatCtx->iformat->name : "";
       if (formatName == "avi") {
         filterName = "mpeg4_unpack_bframes";
-      } else {
-        // For MPEG-4 Part 2 in MP4/MOV/etc., codec configuration (VOS/VOL)
-        // lives only in extradata. Unlike H.264/HEVC there's no codec-specific
-        // BSF that injects it inline, so use dump_extra to prepend extradata
-        // to keyframe packets — the NVCUVID parser needs this to initialize.
-        filterName = "dump_extra";
       }
       break;
     }

src/torchcodec/_core/BetaCudaDeviceInterface.h

@@ -89,6 +89,7 @@ class BetaCudaDeviceInterface : public DeviceInterface {
   CUvideoparser videoParser_ = nullptr;
   UniqueCUvideodecoder decoder_;
   CUVIDEOFORMAT videoFormat_ = {};
+  CUVIDEOFORMATEX parserExtInfo_ = {};
 
   std::queue<CUVIDPARSERDISPINFO> readyFrames_;
 

test/test_decoders.py

@@ -1980,9 +1980,7 @@ def test_default_cuda_interface_backwards(self, asset, seek_mode):
     @pytest.mark.parametrize("seek_mode", ("exact", "approximate"))
     def test_cuda_mpeg4_mp4_first_frame(self, seek_mode):
         # non-regression test for
-        # https://github.com/meta-pytorch/torchcodec/issues/1340.  MPEG-4 Part 2
-        # in MP4 stores codec config (VOS/VOL) only in extradata, never inline.
-        # Without the dump_extra BSF, the NVCUVID parser could not initialize.
+        # https://github.com/meta-pytorch/torchcodec/issues/1340.
         decoder = VideoDecoder(
             TEST_SRC_2_MPEG4_MP4.path, device="cuda", seek_mode=seek_mode
         )

test/utils.py

@@ -729,9 +729,6 @@ def get_empty_chw_tensor(self, *, stream_index: int) -> torch.Tensor:
 )
 
 # ffmpeg -f lavfi -i color=c=black:s=64x64:d=0.034 -c:v mpeg4 -q:v 31 testsrc2_mpeg4.mp4
-# MPEG-4 Part 2 in MP4 stores codec config (VOS/VOL) in extradata only, with
-# no inline copies in the bitstream — exercises the dump_extra BSF path on
-# the default CUDA backend.
 TEST_SRC_2_MPEG4_MP4 = TestVideo(
     filename="testsrc2_mpeg4.mp4",
     default_stream_index=0,