diff mbox

[FFmpeg-devel] hwcontext_d3d11: Fix crash with valid adapter but no device

Message ID c620c5c7-9d9f-ca4c-851c-f94ae8028343@jkqxz.net
State Accepted
Commit 44000b7744a0a3c425bc6d8d924b4efa866aad39
Headers show

Commit Message

Mark Thompson March 27, 2018, 12:22 a.m. UTC 
This crash was introduced by 8bbf2dacbfb4ead1535dea411035994f507f517d,
which could incorrectly overwrite the failure result from creating the
device.

Fixes ticket #7108.
---
 libavutil/hwcontext_d3d11va.c | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

Comments

wm4 March 27, 2018, 3:10 p.m. UTC | #1 
On Tue, 27 Mar 2018 01:22:34 +0100
Mark Thompson <sw@jkqxz.net> wrote:

> This crash was introduced by 8bbf2dacbfb4ead1535dea411035994f507f517d,
> which could incorrectly overwrite the failure result from creating the
> device.
> 
> Fixes ticket #7108.
> ---
>  libavutil/hwcontext_d3d11va.c | 8 +++++---
>  1 file changed, 5 insertions(+), 3 deletions(-)
> 
> diff --git a/libavutil/hwcontext_d3d11va.c b/libavutil/hwcontext_d3d11va.c
> index 960883c9d8..d39fdd3fc8 100644
> --- a/libavutil/hwcontext_d3d11va.c
> +++ b/libavutil/hwcontext_d3d11va.c
> @@ -556,8 +556,6 @@ static int d3d11va_device_create(AVHWDeviceContext *ctx, const char *device,
>          }
>      }
>  
> -    hr = mD3D11CreateDevice(pAdapter, pAdapter ? D3D_DRIVER_TYPE_UNKNOWN : D3D_DRIVER_TYPE_HARDWARE, NULL, creationFlags, NULL, 0,
> -                   D3D11_SDK_VERSION, &device_hwctx->device, NULL, NULL);
>      if (pAdapter) {
>          DXGI_ADAPTER_DESC2 desc;
>          hr = IDXGIAdapter2_GetDesc(pAdapter, &desc);
> @@ -565,8 +563,12 @@ static int d3d11va_device_create(AVHWDeviceContext *ctx, const char *device,
>              av_log(ctx, AV_LOG_INFO, "Using device %04x:%04x (%ls).\n",
>                     desc.VendorId, desc.DeviceId, desc.Description);
>          }
> -        IDXGIAdapter_Release(pAdapter);
>      }
> +
> +    hr = mD3D11CreateDevice(pAdapter, pAdapter ? D3D_DRIVER_TYPE_UNKNOWN : D3D_DRIVER_TYPE_HARDWARE, NULL, creationFlags, NULL, 0,
> +                   D3D11_SDK_VERSION, &device_hwctx->device, NULL, NULL);
> +    if (pAdapter)
> +        IDXGIAdapter_Release(pAdapter);
>      if (FAILED(hr)) {
>          av_log(ctx, AV_LOG_ERROR, "Failed to create Direct3D device (%lx)\n", (long)hr);
>          return AVERROR_UNKNOWN;

LGTM
Mark Thompson March 27, 2018, 11:21 p.m. UTC | #2 
On 27/03/18 16:10, wm4 wrote:
> On Tue, 27 Mar 2018 01:22:34 +0100
> Mark Thompson <sw@jkqxz.net> wrote:
> 
>> This crash was introduced by 8bbf2dacbfb4ead1535dea411035994f507f517d,
>> which could incorrectly overwrite the failure result from creating the
>> device.
>>
>> Fixes ticket #7108.
>> ---
>>  libavutil/hwcontext_d3d11va.c | 8 +++++---
>>  1 file changed, 5 insertions(+), 3 deletions(-)
>>
>> diff --git a/libavutil/hwcontext_d3d11va.c b/libavutil/hwcontext_d3d11va.c
>> index 960883c9d8..d39fdd3fc8 100644
>> --- a/libavutil/hwcontext_d3d11va.c
>> +++ b/libavutil/hwcontext_d3d11va.c
>> @@ -556,8 +556,6 @@ static int d3d11va_device_create(AVHWDeviceContext *ctx, const char *device,
>>          }
>>      }
>>  
>> -    hr = mD3D11CreateDevice(pAdapter, pAdapter ? D3D_DRIVER_TYPE_UNKNOWN : D3D_DRIVER_TYPE_HARDWARE, NULL, creationFlags, NULL, 0,
>> -                   D3D11_SDK_VERSION, &device_hwctx->device, NULL, NULL);
>>      if (pAdapter) {
>>          DXGI_ADAPTER_DESC2 desc;
>>          hr = IDXGIAdapter2_GetDesc(pAdapter, &desc);
>> @@ -565,8 +563,12 @@ static int d3d11va_device_create(AVHWDeviceContext *ctx, const char *device,
>>              av_log(ctx, AV_LOG_INFO, "Using device %04x:%04x (%ls).\n",
>>                     desc.VendorId, desc.DeviceId, desc.Description);
>>          }
>> -        IDXGIAdapter_Release(pAdapter);
>>      }
>> +
>> +    hr = mD3D11CreateDevice(pAdapter, pAdapter ? D3D_DRIVER_TYPE_UNKNOWN : D3D_DRIVER_TYPE_HARDWARE, NULL, creationFlags, NULL, 0,
>> +                   D3D11_SDK_VERSION, &device_hwctx->device, NULL, NULL);
>> +    if (pAdapter)
>> +        IDXGIAdapter_Release(pAdapter);
>>      if (FAILED(hr)) {
>>          av_log(ctx, AV_LOG_ERROR, "Failed to create Direct3D device (%lx)\n", (long)hr);
>>          return AVERROR_UNKNOWN;
> 
> LGTM

Pushed.

Thanks,

- Mark
diff mbox

Patch

diff --git a/libavutil/hwcontext_d3d11va.c b/libavutil/hwcontext_d3d11va.c
index 960883c9d8..d39fdd3fc8 100644
--- a/libavutil/hwcontext_d3d11va.c
+++ b/libavutil/hwcontext_d3d11va.c
@@ -556,8 +556,6 @@  static int d3d11va_device_create(AVHWDeviceContext *ctx, const char *device,
         }
     }
 
-    hr = mD3D11CreateDevice(pAdapter, pAdapter ? D3D_DRIVER_TYPE_UNKNOWN : D3D_DRIVER_TYPE_HARDWARE, NULL, creationFlags, NULL, 0,
-                   D3D11_SDK_VERSION, &device_hwctx->device, NULL, NULL);
     if (pAdapter) {
         DXGI_ADAPTER_DESC2 desc;
         hr = IDXGIAdapter2_GetDesc(pAdapter, &desc);
@@ -565,8 +563,12 @@  static int d3d11va_device_create(AVHWDeviceContext *ctx, const char *device,
             av_log(ctx, AV_LOG_INFO, "Using device %04x:%04x (%ls).\n",
                    desc.VendorId, desc.DeviceId, desc.Description);
         }
-        IDXGIAdapter_Release(pAdapter);
     }
+
+    hr = mD3D11CreateDevice(pAdapter, pAdapter ? D3D_DRIVER_TYPE_UNKNOWN : D3D_DRIVER_TYPE_HARDWARE, NULL, creationFlags, NULL, 0,
+                   D3D11_SDK_VERSION, &device_hwctx->device, NULL, NULL);
+    if (pAdapter)
+        IDXGIAdapter_Release(pAdapter);
     if (FAILED(hr)) {
         av_log(ctx, AV_LOG_ERROR, "Failed to create Direct3D device (%lx)\n", (long)hr);
         return AVERROR_UNKNOWN;