Skip to content

Commit

Permalink
Cherry pick 0.10 fixes to MediaFoundation backend (#200)
Browse files Browse the repository at this point in the history 
* Fix infinite loop when guid_to_frameformat fails (#188)

* Fix bug in MSFoundation backend where media formats are created not exactly as expected by the device and fail to open (#197)

* Fix infinite loop when guid_to_frameformat fails

* Fix bug in MSFoundation backend where media formats are created not exactly as expected by the device and fail to open. Also fix conversion of MSFoundation RGB24, it's RAWBGR instead of RAWRGB

* Fix compilation issue
  • Loading branch information
@eduramiba
eduramiba authored Jan 17, 2025
1 parent 33f0787 commit 5b86e1d
Showing 1 changed file with 121 additions and 105 deletions.
226 changes: 121 additions & 105 deletions nokhwa-bindings-windows/src/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ pub mod wmf {
use nokhwa_core::control::{CameraControl, ControlValueDescription, ControlValue, KnownCameraControl};
use windows::Win32::Media::DirectShow::{CameraControl_Flags_Auto, CameraControl_Flags_Manual};
use windows::Win32::Media::MediaFoundation::{
IMFMediaType, MFCreateSample, MF_SOURCE_READER_FIRST_VIDEO_STREAM,
MFCreateSample, MF_SOURCE_READER_FIRST_VIDEO_STREAM,
};
use windows::{
core::{Interface, GUID, PWSTR},
Expand All @@ -66,14 +66,14 @@ pub mod wmf {
KernelStreaming::GUID_NULL,
MediaFoundation::{
IMFActivate, IMFAttributes, IMFMediaSource, IMFSample, IMFSourceReader,
MFCreateAttributes, MFCreateMediaType, MFCreateSourceReaderFromMediaSource,
MFEnumDeviceSources, MFMediaType_Video, MFShutdown, MFStartup,
MFCreateAttributes, MFCreateSourceReaderFromMediaSource,
MFEnumDeviceSources, MFShutdown, MFStartup,
MFSTARTUP_NOSOCKET, MF_API_VERSION, MF_DEVSOURCE_ATTRIBUTE_FRIENDLY_NAME,
MF_DEVSOURCE_ATTRIBUTE_SOURCE_TYPE,
MF_DEVSOURCE_ATTRIBUTE_SOURCE_TYPE_VIDCAP_GUID,
MF_DEVSOURCE_ATTRIBUTE_SOURCE_TYPE_VIDCAP_SYMBOLIC_LINK, MF_MT_FRAME_RATE,
MF_MT_FRAME_RATE_RANGE_MAX, MF_MT_FRAME_RATE_RANGE_MIN, MF_MT_FRAME_SIZE,
MF_MT_MAJOR_TYPE, MF_MT_SUBTYPE, MF_READWRITE_DISABLE_CONVERTERS,
MF_MT_SUBTYPE, MF_READWRITE_DISABLE_CONVERTERS,
},
},
System::Com::{CoInitializeEx, CoUninitialize, COINIT},
Expand Down Expand Up @@ -166,24 +166,14 @@ pub mod wmf {
fn guid_to_frameformat(guid: GUID) -> Option<FrameFormat> {
match guid {
MF_VIDEO_FORMAT_NV12 => Some(FrameFormat::NV12),
MF_VIDEO_FORMAT_RGB24 => Some(FrameFormat::RAWRGB),
MF_VIDEO_FORMAT_RGB24 => Some(FrameFormat::RAWBGR),
MF_VIDEO_FORMAT_GRAY => Some(FrameFormat::GRAY),
MF_VIDEO_FORMAT_YUY2 => Some(FrameFormat::YUYV),
MF_VIDEO_FORMAT_MJPEG => Some(FrameFormat::MJPEG),
_ => None,
}
}

fn frameformat_to_guid(frameformat: FrameFormat) -> GUID {
match frameformat {
FrameFormat::MJPEG => MF_VIDEO_FORMAT_MJPEG,
FrameFormat::YUYV => MF_VIDEO_FORMAT_YUY2,
FrameFormat::NV12 => MF_VIDEO_FORMAT_NV12,
FrameFormat::GRAY => MF_VIDEO_FORMAT_GRAY,
FrameFormat::RAWRGB => MF_VIDEO_FORMAT_RGB24,
}
}

pub fn initialize_mf() -> Result<(), NokhwaError> {
if !(INITIALIZED.load(Ordering::SeqCst)) {
if let Err(why) = unsafe {
Expand Down Expand Up @@ -575,6 +565,7 @@ pub mod wmf {
self.source_reader
.GetNativeMediaType(MEDIA_FOUNDATION_FIRST_VIDEO_STREAM, index)
} {
index += 1;
let fourcc = match unsafe { media_type.GetGUID(&MF_MT_SUBTYPE) } {
Ok(fcc) => fcc,
Err(why) => {
Expand All @@ -599,7 +590,7 @@ pub mod wmf {
}
};

// MFRatio is represented as 2 u32s in memory. This means we cann convert it to 2
// MFRatio is represented as 2 u32s in memory. This means we can convert it to 2
let framerate_list = {
let mut framerates = vec![0_u32; 3];
if let Ok(fraction_u64) =
Expand All @@ -612,16 +603,6 @@ pub mod wmf {
}
framerates.push(numerator);
};
if let Ok(fraction_u64) =
unsafe { media_type.GetUINT64(&MF_MT_FRAME_RATE_RANGE_MAX) }
{
let mut numerator = (fraction_u64 >> 32) as u32;
let denominator = fraction_u64 as u32;
if denominator != 1 {
numerator = 0;
}
framerates.push(numerator);
};
if let Ok(fraction_u64) = unsafe { media_type.GetUINT64(&MF_MT_FRAME_RATE) } {
let mut numerator = (fraction_u64 >> 32) as u32;
let denominator = fraction_u64 as u32;
Expand Down Expand Up @@ -660,8 +641,6 @@ pub mod wmf {
));
}
}

index += 1;
}
Ok(camera_format_list)
}
Expand Down Expand Up @@ -1016,88 +995,125 @@ pub mod wmf {
}

pub fn set_format(&mut self, format: CameraFormat) -> Result<(), NokhwaError> {
// convert to media_type
let media_type: IMFMediaType = match unsafe { MFCreateMediaType() } {
Ok(mt) => mt,
Err(why) => {
return Err(NokhwaError::StructureError {
structure: "IMFMediaType".to_string(),
error: why.to_string(),
})
// We need to make sure to use all the original attributes of the IMFMediaType to avoid problems.
// Otherwise, constructing IMFMediaType from scratch can sometimes fail due to not exactly matching.
// Therefore, we search for the first media_type that matches and also works correctly.

let mut last_error : Option<NokhwaError> = None;

let mut index = 0;
while let Ok(media_type) = unsafe {
self.source_reader
.GetNativeMediaType(MEDIA_FOUNDATION_FIRST_VIDEO_STREAM, index)
} {
index += 1;
let fourcc = match unsafe { media_type.GetGUID(&MF_MT_SUBTYPE) } {
Ok(fcc) => fcc,
Err(why) => {
return Err(NokhwaError::GetPropertyError {
property: "MF_MT_SUBTYPE".to_string(),
error: why.to_string(),
})
}
};

let frame_fmt = match guid_to_frameformat(fourcc) {
Some(fcc) => fcc,
None => continue,
};

if frame_fmt != format.format() {
continue;
}
};

// set relevant things
let resolution = (u64::from(format.resolution().width_x) << 32_u64)
+ u64::from(format.resolution().height_y);
let fps = {
let frame_rate_u64 = 0_u64;
let mut bytes: [u8; 8] = frame_rate_u64.to_le_bytes();
bytes[7] = format.frame_rate() as u8;
bytes[3] = 0x01;
u64::from_le_bytes(bytes)
};
let fourcc = frameformat_to_guid(format.format());
// setting to the new media_type
if let Err(why) = unsafe { media_type.SetGUID(&MF_MT_MAJOR_TYPE, &MFMediaType_Video) } {
return Err(NokhwaError::SetPropertyError {
property: "MF_MT_MAJOR_TYPE".to_string(),
value: "MFMediaType_Video".to_string(),
error: why.to_string(),
});
}
if let Err(why) = unsafe { media_type.SetGUID(&MF_MT_SUBTYPE, &fourcc) } {
return Err(NokhwaError::SetPropertyError {
property: "MF_MT_SUBTYPE".to_string(),
value: format!("{:?}", fourcc),
error: why.to_string(),
});
}
if let Err(why) = unsafe { media_type.SetUINT64(&MF_MT_FRAME_SIZE, resolution) } {
return Err(NokhwaError::SetPropertyError {
property: "MF_MT_FRAME_SIZE".to_string(),
value: resolution.to_string(),
error: why.to_string(),
});
}
if let Err(why) = unsafe { media_type.SetUINT64(&MF_MT_FRAME_RATE, fps) } {
return Err(NokhwaError::SetPropertyError {
property: "MF_MT_FRAME_RATE".to_string(),
value: fps.to_string(),
error: why.to_string(),
});
}
if let Err(why) = unsafe { media_type.SetUINT64(&MF_MT_FRAME_RATE_RANGE_MIN, fps) } {
return Err(NokhwaError::SetPropertyError {
property: "MF_MT_FRAME_RATE_RANGE_MIN".to_string(),
value: fps.to_string(),
error: why.to_string(),
});
}
if let Err(why) = unsafe { media_type.SetUINT64(&MF_MT_FRAME_RATE_RANGE_MAX, fps) } {
return Err(NokhwaError::SetPropertyError {
property: "MF_MT_FRAME_RATE_RANGE_MAX".to_string(),
value: fps.to_string(),
error: why.to_string(),
});
let (width, height) = match unsafe { media_type.GetUINT64(&MF_MT_FRAME_SIZE) } {
Ok(res_u64) => {
let width = (res_u64 >> 32) as u32;
let height = res_u64 as u32; // the cast will truncate the upper bits
(width, height)
}
Err(why) => {
return Err(NokhwaError::GetPropertyError {
property: "MF_MT_FRAME_SIZE".to_string(),
error: why.to_string(),
})
}
};

if (Resolution { width_x: width, height_y: height }) != format.resolution() {
continue;
}

// MFRatio is represented as 2 u32s in memory. This means we can convert it to 2
let framerate_list = {
let mut framerates = vec![0_u32; 3];
if let Ok(fraction_u64) =
unsafe { media_type.GetUINT64(&MF_MT_FRAME_RATE_RANGE_MAX) }
{
let mut numerator = (fraction_u64 >> 32) as u32;
let denominator = fraction_u64 as u32;
if denominator != 1 {
numerator = 0;
}
framerates.push(numerator);
};
if let Ok(fraction_u64) = unsafe { media_type.GetUINT64(&MF_MT_FRAME_RATE) } {
let mut numerator = (fraction_u64 >> 32) as u32;
let denominator = fraction_u64 as u32;
if denominator != 1 {
numerator = 0;
}
framerates.push(numerator);
};
if let Ok(fraction_u64) =
unsafe { media_type.GetUINT64(&MF_MT_FRAME_RATE_RANGE_MIN) }
{
let mut numerator = (fraction_u64 >> 32) as u32;
let denominator = fraction_u64 as u32;
if denominator != 1 {
numerator = 0;
}
framerates.push(numerator);
};
framerates
};

for frame_rate in framerate_list {
if frame_rate == format.frame_rate() {
let result = unsafe {
self.source_reader.SetCurrentMediaType(
MEDIA_FOUNDATION_FIRST_VIDEO_STREAM,
None,
&media_type,
)
};

match result {
Ok(_) => {
self.device_format = format;
self.format_refreshed()?;
return Ok(());
},
Err(why) => {
last_error = Some(NokhwaError::SetPropertyError {
property: "MEDIA_FOUNDATION_FIRST_VIDEO_STREAM".to_string(),
value: format!("{media_type:?}"),
error: why.to_string(),
});
}
}
}
}
}

if let Err(why) = unsafe {
self.source_reader.SetCurrentMediaType(
MEDIA_FOUNDATION_FIRST_VIDEO_STREAM,
None,
&media_type,
)
} {
return Err(NokhwaError::SetPropertyError {
property: "MEDIA_FOUNDATION_FIRST_VIDEO_STREAM".to_string(),
value: format!("{media_type:?}"),
error: why.to_string(),
});
if let Some(err) = last_error {
return Err(err);
}
self.device_format = format;
self.format_refreshed()?;
Ok(())

Err(NokhwaError::InitializeError {
backend: ApiBackend::MediaFoundation,
error: "Failed to fulfill requested format".to_string(),
})
}

pub fn is_stream_open(&self) -> bool {
Expand Down

0 comments on commit 5b86e1d

Please sign in to comment.