vd_lavc, vdpau, vaapi: restore emulated API avoidance

This code is for trying to avoid using an emulation layer when using
auto probing, so that we end up using the actual API the drivers
provide. It was destroyed in the recent refactor.

video/decode/vd_lavc.c

@@ -458,6 +458,17 @@ static void select_and_set_hwdec(struct dec_video *vd)
                     MP_VERBOSE(vd, "Could not create device.\n");
                     continue;
                 }
+
+                const struct hwcontext_fns *fns =
+                            hwdec_get_hwcontext_fns(hwdec->lavc_device);
+                if (fns && fns->is_emulated && fns->is_emulated(ctx->hwdec_dev)) {
+                    if (hwdec_auto) {
+                        MP_VERBOSE(vd, "Not using emulated API.\n");
+                        av_buffer_unref(&ctx->hwdec_dev);
+                        continue;
+                    }
+                    MP_WARN(vd, "Using emulated hardware decoding API.\n");
+                }
             }
 
             ctx->use_hwdec = true;

video/hwdec.h

@@ -119,6 +119,8 @@ struct hwcontext_fns {
     struct AVBufferRef *(*create_dev)(struct mpv_global *global,
                                       struct mp_log *log,
                                       struct hwcontext_create_dev_params *params);
+    // Return whether this is using some sort of sub-optimal emulation layer.
+    bool (*is_emulated)(struct AVBufferRef *hw_device_ctx);
 };
 
 // The parameter is of type enum AVHWDeviceType (as in int to avoid extensive

video/vaapi.c

@@ -204,12 +204,21 @@ VASurfaceID va_surface_id(struct mp_image *mpi)
         (VASurfaceID)(uintptr_t)mpi->planes[3] : VA_INVALID_ID;
 }
 
-bool va_guess_if_emulated(struct mp_vaapi_ctx *ctx)
+static bool is_emulated(struct AVBufferRef *hw_device_ctx)
 {
-    const char *s = vaQueryVendorString(ctx->display);
+    AVHWDeviceContext *hwctx = (void *)hw_device_ctx->data;
+    AVVAAPIDeviceContext *vactx = hwctx->hwctx;
+
+    const char *s = vaQueryVendorString(vactx->display);
     return s && strstr(s, "VDPAU backend");
 }
 
+
+bool va_guess_if_emulated(struct mp_vaapi_ctx *ctx)
+{
+    return is_emulated(ctx->av_device_ref);
+}
+
 struct va_native_display {
     void (*create)(VADisplay **out_display, void **out_native_ctx);
     void (*destroy)(void *native_ctx);
@@ -327,4 +336,5 @@ static struct AVBufferRef *va_create_standalone(struct mpv_global *global,
 const struct hwcontext_fns hwcontext_fns_vaapi = {
     .av_hwdevice_type = AV_HWDEVICE_TYPE_VAAPI,
     .create_dev = va_create_standalone,
+    .is_emulated = is_emulated,
 };

video/vdpau.c

@@ -537,6 +537,26 @@ bool mp_vdpau_guess_if_emulated(struct mp_vdpau_ctx *ctx)
     return vdp_st == VDP_STATUS_OK && info && strstr(info, "VAAPI");
 }
 
+// (This clearly works only for contexts wrapped by our code.)
+struct mp_vdpau_ctx *mp_vdpau_get_ctx_from_av(AVBufferRef *hw_device_ctx)
+{
+    AVHWDeviceContext *hwctx = (void *)hw_device_ctx->data;
+
+    if (hwctx->free != free_device_ref)
+        return NULL; // not ours
+
+    return hwctx->user_opaque;
+}
+
+static bool is_emulated(struct AVBufferRef *hw_device_ctx)
+{
+    struct mp_vdpau_ctx *ctx = mp_vdpau_get_ctx_from_av(hw_device_ctx);
+    if (!ctx)
+        return false;
+
+    return mp_vdpau_guess_if_emulated(ctx);
+}
+
 static struct AVBufferRef *vdpau_create_standalone(struct mpv_global *global,
         struct mp_log *log, struct hwcontext_create_dev_params *params)
 {
@@ -555,11 +575,11 @@ static struct AVBufferRef *vdpau_create_standalone(struct mpv_global *global,
 
     vdp->hwctx.emulated = mp_vdpau_guess_if_emulated(vdp);
     vdp->close_display = true;
-    mp_warn(log, "idk\n");
     return vdp->hwctx.av_device_ref;
 }
 
 const struct hwcontext_fns hwcontext_fns_vdpau = {
     .av_hwdevice_type = AV_HWDEVICE_TYPE_VDPAU,
     .create_dev = vdpau_create_standalone,
+    .is_emulated = is_emulated,
 };

video/vdpau.h

@@ -103,6 +103,8 @@ bool mp_vdpau_get_rgb_format(int imgfmt, VdpRGBAFormat *out_rgba_format);
 struct mp_image *mp_vdpau_upload_video_surface(struct mp_vdpau_ctx *ctx,
                                                struct mp_image *mpi);
 
+struct mp_vdpau_ctx *mp_vdpau_get_ctx_from_av(struct AVBufferRef *hw_device_ctx);
+
 bool mp_vdpau_guess_if_emulated(struct mp_vdpau_ctx *ctx);
 
 #endif