GS Capture: Allow selecting the pixel format based on the current codec

pcsx2-qt/Settings/GraphicsSettingsWidget.cpp

@@ -397,6 +397,7 @@ GraphicsSettingsWidget::GraphicsSettingsWidget(SettingsWindow* dialog, QWidget*
 			&GraphicsSettingsWidget::onEnableAudioCaptureArgumentsChanged);
 
 		onCaptureContainerChanged();
+		onCaptureCodecChanged();
 		onEnableVideoCaptureChanged();
 		onEnableVideoCaptureArgumentsChanged();
 		onVideoCaptureAutoResolutionChanged();
@@ -735,6 +736,10 @@ GraphicsSettingsWidget::GraphicsSettingsWidget(SettingsWindow* dialog, QWidget*
 		
 		"<b>If unsure, leave it on default.<b>"));
 
+		dialog->registerWidgetHelp(m_ui.videoCaptureFormat, tr("Video Format"), tr("Default"), tr("Selects which Video Format to be used for Video Capture. If by chance the codec does not support the format, the first format available will be used. "
+		
+		"<b>If unsure, leave it on default.<b>"));
+
 		dialog->registerWidgetHelp(m_ui.videoCaptureBitrate, tr("Video Bitrate"), tr("6000 kbps"), tr("Sets the video bitrate to be used. "
 		
 		"Larger bitrate generally yields better video quality at the cost of larger resulting file size."));
@@ -914,6 +919,7 @@ void GraphicsSettingsWidget::onCaptureContainerChanged()
 
 	SettingWidgetBinder::BindWidgetToStringSetting(
 		m_dialog->getSettingsInterface(), m_ui.videoCaptureCodec, "EmuCore/GS", "VideoCaptureCodec");
+	connect(m_ui.videoCaptureCodec, &QComboBox::currentIndexChanged, this, &GraphicsSettingsWidget::onCaptureCodecChanged);
 
 	m_ui.audioCaptureCodec->disconnect();
 	m_ui.audioCaptureCodec->clear();
@@ -929,6 +935,30 @@ void GraphicsSettingsWidget::onCaptureContainerChanged()
 		m_dialog->getSettingsInterface(), m_ui.audioCaptureCodec, "EmuCore/GS", "AudioCaptureCodec");
 }
 
+void GraphicsSettingsWidget::GraphicsSettingsWidget::onCaptureCodecChanged()
+{
+	m_ui.videoCaptureFormat->disconnect();
+	m_ui.videoCaptureFormat->clear();
+	//: This string refers to a default pixel format
+	m_ui.videoCaptureFormat->addItem(tr("Default"), "");
+
+	const std::string codec(
+		m_dialog->getEffectiveStringValue("EmuCore/GS", "VideoCaptureCodec", ""));
+
+	if (!codec.empty())
+	{
+		for (const auto& [id, name] : GSCapture::GetVideoFormatList(codec.c_str()))
+		{
+			const QString qid(QString::number(id));
+			const QString qname(QString::fromStdString(name));
+			m_ui.videoCaptureFormat->addItem(qname, qid);
+		}
+	}
+
+	SettingWidgetBinder::BindWidgetToStringSetting(
+		m_dialog->getSettingsInterface(), m_ui.videoCaptureFormat, "EmuCore/GS", "VideoCaptureFormat");
+}
+
 void GraphicsSettingsWidget::onEnableVideoCaptureChanged()
 {
 	const bool enabled = m_dialog->getEffectiveBoolValue("EmuCore/GS", "EnableVideoCapture", true);

pcsx2-qt/Settings/GraphicsSettingsWidget.h

@@ -38,6 +38,7 @@ private Q_SLOTS:
 	void onTextureReplacementChanged();
 	void onShadeBoostChanged();
 	void onCaptureContainerChanged();
+	void onCaptureCodecChanged();
 	void onEnableVideoCaptureChanged();
 	void onEnableVideoCaptureArgumentsChanged();
 	void onVideoCaptureAutoResolutionChanged();

pcsx2-qt/Settings/GraphicsSettingsWidget.ui

@@ -1847,13 +1847,23 @@
                 <widget class="QComboBox" name="videoCaptureCodec"/>
                </item>
                <item row="1" column="0">
+                <widget class="QLabel" name="videoCaptureFomatLabel">
+                 <property name="text">
+                  <string>Format:</string>
+                 </property>
+                </widget>
+               </item>
+               <item row="1" column="1">
+                <widget class="QComboBox" name="videoCaptureFormat"/>
+               </item>
+               <item row="2" column="0">
                 <widget class="QLabel" name="videoCaptureBitrateLabel">
                  <property name="text">
                   <string>Bitrate:</string>
                  </property>
                 </widget>
                </item>
-               <item row="1" column="1">
+               <item row="2" column="1">
                 <widget class="QSpinBox" name="videoCaptureBitrate">
                  <property name="suffix">
                   <string extracomment="Unit that will appear next to a number. Alter the space or whatever is needed before the text depending on your language."> kbps</string>
@@ -1869,14 +1879,14 @@
                  </property>
                 </widget>
                </item>
-               <item row="2" column="0">
+               <item row="3" column="0">
                 <widget class="QLabel" name="videoCaptureResolutionLabel">
                  <property name="text">
                   <string>Resolution:</string>
                  </property>
                 </widget>
                </item>
-               <item row="2" column="1">
+               <item row="3" column="1">
                 <layout class="QHBoxLayout" name="horizontalLayout_10" stretch="1,0,1,0">
                  <item>
                   <widget class="QSpinBox" name="videoCaptureWidth">
@@ -1926,14 +1936,14 @@
                  </item>
                 </layout>
                </item>
-               <item row="3" column="0" colspan="2">
+               <item row="4" column="0" colspan="2">
                 <widget class="QCheckBox" name="enableVideoCaptureArguments">
                  <property name="text">
                   <string>Extra Arguments</string>
                  </property>
                 </widget>
                </item>
-               <item row="4" column="0" colspan="2">
+               <item row="5" column="0" colspan="2">
                 <widget class="QLineEdit" name="videoCaptureArguments"/>
                </item>
               </layout>

pcsx2/Config.h

@@ -731,6 +731,7 @@ struct Pcsx2Config
 
 		std::string CaptureContainer = DEFAULT_CAPTURE_CONTAINER;
 		std::string VideoCaptureCodec;
+		std::string VideoCaptureFormat;
 		std::string VideoCaptureParameters;
 		std::string AudioCaptureCodec;
 		std::string AudioCaptureParameters;

pcsx2/GS/GSCapture.cpp

@@ -43,6 +43,7 @@ extern "C" {
 #include "libavutil/opt.h"
 #include "libavutil/version.h"
 #include "libavutil/channel_layout.h"
+#include "libavutil/pixdesc.h"
 #include "libswscale/swscale.h"
 #include "libswscale/version.h"
 #include "libswresample/swresample.h"
@@ -117,7 +118,8 @@ extern "C" {
 	X(av_hwframe_transfer_data) \
 	X(av_hwframe_get_buffer) \
 	X(av_buffer_ref) \
-	X(av_buffer_unref)
+	X(av_buffer_unref) \
+	X(av_get_pix_fmt_name)
 
 #define VISIT_SWSCALE_IMPORTS(X) \
 	X(sws_getCachedContext) \
@@ -447,25 +449,30 @@ bool GSCapture::BeginCapture(float fps, GSVector2i recommendedResolution, float
 		wrap_av_reduce(&s_video_codec_context->time_base.num, &s_video_codec_context->time_base.den, 10000,
 			static_cast<s64>(static_cast<double>(fps) * 10000.0), std::numeric_limits<s32>::max());
 
-		// Default to YUV 4:2:0 if the codec doesn't specify a pixel format.
-		AVPixelFormat sw_pix_fmt = AV_PIX_FMT_YUV420P;
+		// Default to NV12 if not overridden by the user
+		const AVPixelFormat preferred_sw_pix_fmt = GSConfig.VideoCaptureFormat.empty() ? AV_PIX_FMT_NV12 : static_cast<AVPixelFormat>(std::stoi(GSConfig.VideoCaptureFormat));
+		AVPixelFormat sw_pix_fmt = preferred_sw_pix_fmt;
 		if (vcodec->pix_fmts)
 		{
-			// Prefer YUV420 given the choice, but otherwise fall back to whatever it supports.
 			sw_pix_fmt = vcodec->pix_fmts[0];
 			for (u32 i = 0; vcodec->pix_fmts[i] != AV_PIX_FMT_NONE; i++)
 			{
-				if (vcodec->pix_fmts[i] == AV_PIX_FMT_YUV420P)
+				if (vcodec->pix_fmts[i] == preferred_sw_pix_fmt)
 				{
 					sw_pix_fmt = vcodec->pix_fmts[i];
 					break;
 				}
 			}
 		}
+
 		if (sw_pix_fmt == AV_PIX_FMT_VAAPI)
 			sw_pix_fmt = AV_PIX_FMT_NV12;
+
 		s_video_codec_context->pix_fmt = sw_pix_fmt;
 
+		if (preferred_sw_pix_fmt != sw_pix_fmt)
+			Console.Warning("GSCapture: preferred pixel format (%d) was unsupported by the codec. Using (%d) instead.", preferred_sw_pix_fmt, sw_pix_fmt);
+
 		// Can we use hardware encoding?
 		const AVCodecHWConfig* hwconfig = wrap_avcodec_get_hw_config(vcodec, 0);
 		if (hwconfig && hwconfig->pix_fmt != AV_PIX_FMT_NONE && hwconfig->pix_fmt != sw_pix_fmt)
@@ -692,11 +699,11 @@ bool GSCapture::BeginCapture(float fps, GSVector2i recommendedResolution, float
 				return false;
 			}
 
-			#if LIBAVUTIL_VERSION_MAJOR >= 57
-				const AVChannelLayout layout = AV_CHANNEL_LAYOUT_STEREO;
-				wrap_av_opt_set_chlayout(s_swr_context, "in_chlayout", &layout, 0);
-				wrap_av_opt_set_chlayout(s_swr_context, "out_chlayout", &layout, 0);
-			#endif
+#if LIBAVUTIL_VERSION_MAJOR >= 57
+			const AVChannelLayout layout = AV_CHANNEL_LAYOUT_STEREO;
+			wrap_av_opt_set_chlayout(s_swr_context, "in_chlayout", &layout, 0);
+			wrap_av_opt_set_chlayout(s_swr_context, "out_chlayout", &layout, 0);
+#endif
 
 			wrap_av_opt_set_int(s_swr_context, "in_channel_count", AUDIO_CHANNELS, 0);
 			wrap_av_opt_set_int(s_swr_context, "in_sample_rate", sample_rate, 0);
@@ -1502,3 +1509,33 @@ GSCapture::CodecList GSCapture::GetAudioCodecList(const char* container)
 {
 	return GetCodecListForContainer(container, AVMEDIA_TYPE_AUDIO);
 }
+
+GSCapture::FormatList GSCapture::GetVideoFormatList(const char* codec)
+{
+	FormatList ret;
+
+	if (!LoadFFmpeg(false))
+		return ret;
+
+	const AVCodec* v_codec = wrap_avcodec_find_encoder_by_name(codec);
+
+	if (!v_codec)
+	{
+		Console.Error("(GetVideoFormatList) avcodec_find_encoder_by_name() failed");
+		return ret;
+	}
+
+	// rawvideo doesn't have a list of formats.
+	if(v_codec->pix_fmts == nullptr)
+	{
+		Console.Error("(GetVideoFormatList) v_codec->pix_fmts is null.");
+		return ret;
+	}
+
+	for (int i = 0; v_codec->pix_fmts[i] != AVPixelFormat::AV_PIX_FMT_NONE; i++)
+	{
+		ret.emplace_back(v_codec->pix_fmts[i], wrap_av_get_pix_fmt_name(v_codec->pix_fmts[i]));
+	}
+
+	return ret;
+}

pcsx2/GS/GSCapture.h

@@ -37,4 +37,8 @@ namespace GSCapture
 	using CodecList = std::vector<CodecName>;
 	CodecList GetVideoCodecList(const char* container);
 	CodecList GetAudioCodecList(const char* container);
+
+	using FormatName = std::pair<int , std::string>; // id,name
+	using FormatList = std::vector<FormatName>;
+	FormatList GetVideoFormatList(const char* codec);
 }; // namespace GSCapture

pcsx2/Pcsx2Config.cpp

@@ -750,6 +750,7 @@ bool Pcsx2Config::GSOptions::OptionsAreEqual(const GSOptions& right) const
 
 		OpEqu(CaptureContainer) &&
 		OpEqu(VideoCaptureCodec) &&
+		OpEqu(VideoCaptureFormat) &&
 		OpEqu(VideoCaptureParameters) &&
 		OpEqu(AudioCaptureCodec) &&
 		OpEqu(AudioCaptureParameters) &&
@@ -925,6 +926,7 @@ void Pcsx2Config::GSOptions::LoadSave(SettingsWrapper& wrap)
 
 	SettingsWrapEntryEx(CaptureContainer, "CaptureContainer");
 	SettingsWrapEntryEx(VideoCaptureCodec, "VideoCaptureCodec");
+	SettingsWrapEntryEx(VideoCaptureFormat, "VideoCaptureFormat");
 	SettingsWrapEntryEx(VideoCaptureParameters, "VideoCaptureParameters");
 	SettingsWrapEntryEx(AudioCaptureCodec, "AudioCaptureCodec");
 	SettingsWrapEntryEx(AudioCaptureParameters, "AudioCaptureParameters");