vd_lavc: derive hwdec surface pool size from VO requirements

Change --hwdec-extra-frames default to auto, which sizes fixed hw frame
pools from the number of frames the VO declares to reference, instead of
a static guess. Fixes pool exhaustion with e.g. --interpolation and a
tscale filter with larger radius.

Fixes: #18205

Author: kasper93

DOCS/interface-changes/hwdec-extra-frames-auto.txt

@@ -0,0 +1 @@
+change `--hwdec-extra-frames` default to `auto`, sized from VO frame retention

DOCS/man/options.rst

@@ -1569,20 +1569,27 @@ Video
     Runtime changes to this are ignored (the current option value is used
     whenever the renderer is created).
 
-``--hwdec-extra-frames=<N>``
-    Number of GPU frames hardware decoding should preallocate (default: see
-    ``--list-options`` output). If this is too low, frame allocation may fail
-    during decoding, and video frames might get dropped and/or corrupted.
-    Setting it too high simply wastes GPU memory and has no advantages.
+``--hwdec-extra-frames=<auto|N>``
+    Number of extra GPU frames hardware decoding should preallocate, on top of
+    what the codec itself requires (default: ``auto``).
 
     This value is used only for hardware decoding APIs which require
     preallocating surfaces (known examples include ``d3d11va`` and ``vaapi``).
     For other APIs, frames are allocated as needed. The details depend on the
     libavcodec implementations of the hardware decoders.
 
-    The required number of surfaces depends on dynamic runtime situations. The
-    default is a fixed value that is thought to be sufficient for most uses. But
-    in certain situations, it may not be enough.
+    In ``auto`` mode, the number is derived from how many frames the player
+    and the VO may reference at the same time. For example, enabling
+    ``--interpolation`` increases it in proportion to the ``--tscale`` filter
+    radius, and VO deinterlacing adds the temporal reference frames it needs.
+    The value is determined when the decoder is initialized; runtime option
+    changes that increase frame requirements do not resize the pool, so set a
+    fixed value if you intend to switch such options during playback.
+
+    Setting a fixed value overrides the automatic sizing in both directions.
+    If it is too low, frame allocation may fail during decoding, and video
+    frames might get dropped and/or corrupted. Setting it too high simply
+    wastes GPU memory and has no advantages.
 
 ``--hwdec-image-format=<name>``
     Set the internal pixel format used by hardware decoding via ``--hwdec``
@@ -1822,6 +1829,10 @@ Video
     inserted deinterlacing filters, and that this will make video look worse if
     it's not actually interlaced.
 
+    Enabling video output deinterlacing at runtime may require setting
+    ``--hwdec-extra-frames``, as the hardware decoder's surface pool is sized
+    at decoder initialization.
+
 ``--deinterlace-field-parity=<tff|bff|auto>``
     Specify the field parity/order when deinterlacing (default: auto).
     Each frame of an interlaced video is divided into two fields, which are
@@ -5984,6 +5995,10 @@ them.
     being the smoothest/blurriest and ``oversample`` being the sharpest/least
     smooth.
 
+    Switching to a filter with a larger radius at runtime may require setting
+    ``--hwdec-extra-frames``, as the hardware decoder's surface pool is sized
+    at decoder initialization.
+
 ``--scale-param1=<value>``, ``--scale-param2=<value>``, ``--cscale-param1=<value>``, ``--cscale-param2=<value>``, ``--dscale-param1=<value>``, ``--dscale-param2=<value>``, ``--tscale-param1=<value>``, ``--tscale-param2=<value>``
     Set filter parameters. By default, these are set to the special string
     ``default``, which maps to a scaler-specific default value. Ignored if the
@@ -6136,6 +6151,9 @@ them.
     the video along the temporal axis. The filter used can be controlled using
     the ``--tscale`` setting.
 
+    Enabling this at runtime may require setting ``--hwdec-extra-frames``, as
+    the hardware decoder's surface pool is sized at decoder initialization.
+
 ``--interpolation-threshold=<0..1,-1>``
     Threshold below which frame ratio interpolation gets disabled (default:
     ``0.01``). This is calculated as ``abs(disphz/vfps - 1) < threshold``,

video/decode/vd_lavc.c

@@ -68,6 +68,13 @@ static int hwdec_opt_help(struct mp_log *log, const m_option_t *opt,
 #define HWDEC_DELAY_QUEUE_COUNT 2
 #define HWDEC_WAIT_KEYFRAME_COUNT 96
 
+// Fallback number of extra surfaces for fixed-size hw frame pools, if the
+// actual requirement cannot be determined.
+#define HWDEC_EXTRA_FRAMES 6
+
+// Surfaces referenced outside of libavcodec on top of what the VO declares
+#define HWDEC_EXTRA_INFLIGHT_FRAMES 3
+
 #define OPT_BASE_STRUCT struct vd_lavc_params
 
 struct vd_lavc_params {
@@ -151,8 +158,8 @@ const struct m_sub_options hwdec_conf = {
             .flags = M_OPT_OPTIONAL_PARAM | M_OPT_ALLOW_NO | UPDATE_HWDEC},
         {"hwdec-codecs", OPT_STRING(hwdec_codecs),
             .flags = UPDATE_HWDEC},
-        {"hwdec-extra-frames", OPT_INT(hwdec_extra_frames), M_RANGE(0, 256),
-            .flags = UPDATE_VD},
+        {"hwdec-extra-frames", OPT_CHOICE(hwdec_extra_frames, {"auto", -1}),
+            M_RANGE(0, 256), .flags = UPDATE_VD},
         {"hwdec-image-format", OPT_IMAGEFORMAT(hwdec_image_format),
             .flags = UPDATE_VO},
         {"hwdec-software-fallback", OPT_CHOICE(software_fallback,
@@ -167,11 +174,7 @@ const struct m_sub_options hwdec_conf = {
         .software_fallback = 3,
         .hwdec_api = (char *[]){"no", NULL,},
         .hwdec_codecs = "h264,vc1,hevc,vp8,vp9,av1,prores,prores_raw,ffv1,dpx,apv",
-        // Maximum number of surfaces the player wants to buffer. This number
-        // might require adjustment depending on whatever the player does;
-        // for example, if vo_gpu increases the number of reference surfaces for
-        // interpolation, this value has to be increased too.
-        .hwdec_extra_frames = 6,
+        .hwdec_extra_frames = -1,
         .hwdec_threads = 4,
     },
 };
@@ -946,6 +949,24 @@ static void uninit_avctx(struct mp_filter *vd)
     ctx->use_hwdec = false;
 }
 
+// Number of extra surfaces to allocate on top of what libavcodec computed
+// for the codec itself, i.e. how many frames the rest of the player may hold
+// on to at any given time.
+static int hwdec_get_extra_frames(vd_ffmpeg_ctx *ctx)
+{
+    int extra = ctx->hwdec_opts->hwdec_extra_frames;
+    if (extra >= 0)
+        return extra;
+
+    extra = HWDEC_EXTRA_FRAMES;
+    // With native hwdec, every frame queued towards or retained by the VO
+    // pins a surface from the fixed pool. Copying wrappers release surfaces
+    // right after download, the fallback is enough for them.
+    if (!ctx->hwdec.copying && ctx->vo)
+        extra = MPMAX(extra, vo_get_num_frame_refs(ctx->vo) + HWDEC_EXTRA_INFLIGHT_FRAMES);
+    return extra;
+}
+
 static int init_generic_hwaccel(struct AVCodecContext *avctx, enum AVPixelFormat hw_fmt)
 {
     struct mp_filter *vd = avctx->opaque;
@@ -975,7 +996,7 @@ static int init_generic_hwaccel(struct AVCodecContext *avctx, enum AVPixelFormat
     // 1 surface is already included by libavcodec. The field is 0 if the
     // hwaccel supports dynamic surface allocation.
     if (new_fctx->initial_pool_size)
-        new_fctx->initial_pool_size += ctx->hwdec_opts->hwdec_extra_frames - 1;
+        new_fctx->initial_pool_size += hwdec_get_extra_frames(ctx) - 1;
 
     const struct hwcontext_fns *fns =
         hwdec_get_hwcontext_fns(new_fctx->device_ctx->type);
@@ -1001,6 +1022,11 @@ static int init_generic_hwaccel(struct AVCodecContext *avctx, enum AVPixelFormat
             goto error;
         }
 
+        if (new_fctx->initial_pool_size) {
+            MP_VERBOSE(ctx, "Allocated a fixed pool of %d hw surfaces.\n",
+                       new_fctx->initial_pool_size);
+        }
+
         ctx->cached_hw_frames_ctx = new_frames_ctx;
         new_frames_ctx = NULL;
     }