FFmpeg: Improve initialization reliability and cleanup

CHANGES

@@ -35,6 +35,7 @@ Other fixes:
  - All: Fix several memory leaks
  - LR35902: Fix trailing whitespace in disassembly
  - Qt: Fix adjusting magnification in tile viewer when not fitting to window
+ - FFmpeg: Improve initialization reliability and cleanup
 Misc:
  - GBA Savedata: EEPROM performance fixes
  - GBA Savedata: Automatically map 1Mbit Flash files as 1Mbit Flash

src/feature/ffmpeg/ffmpeg-encoder.c

@@ -42,19 +42,25 @@ void FFmpegEncoderInit(struct FFmpegEncoder* encoder) {
 	encoder->d.postAudioFrame = _ffmpegPostAudioFrame;
 	encoder->d.postAudioBuffer = 0;
 
-	encoder->audioCodec = 0;
-	encoder->videoCodec = 0;
-	encoder->containerFormat = 0;
+	encoder->audioCodec = NULL;
+	encoder->videoCodec = NULL;
+	encoder->containerFormat = NULL;
 	FFmpegEncoderSetAudio(encoder, "flac", 0);
 	FFmpegEncoderSetVideo(encoder, "png", 0);
 	FFmpegEncoderSetContainer(encoder, "matroska");
 	FFmpegEncoderSetDimensions(encoder, GBA_VIDEO_HORIZONTAL_PIXELS, GBA_VIDEO_VERTICAL_PIXELS);
 	encoder->iwidth = GBA_VIDEO_HORIZONTAL_PIXELS;
 	encoder->iheight = GBA_VIDEO_VERTICAL_PIXELS;
-	encoder->resampleContext = 0;
-	encoder->absf = 0;
-	encoder->context = 0;
+	encoder->resampleContext = NULL;
+	encoder->absf = NULL;
+	encoder->context = NULL;
 	encoder->scaleContext = NULL;
+	encoder->audioStream = NULL;
+	encoder->audioFrame = NULL;
+	encoder->audioBuffer = NULL;
+	encoder->postaudioBuffer = NULL;
+	encoder->videoStream = NULL;
+	encoder->videoFrame = NULL;
 }
 
 bool FFmpegEncoderSetAudio(struct FFmpegEncoder* encoder, const char* acodec, unsigned abr) {
@@ -198,7 +204,11 @@ bool FFmpegEncoderVerifyContainer(struct FFmpegEncoder* encoder) {
 bool FFmpegEncoderOpen(struct FFmpegEncoder* encoder, const char* outfile) {
 	AVCodec* acodec = avcodec_find_encoder_by_name(encoder->audioCodec);
 	AVCodec* vcodec = avcodec_find_encoder_by_name(encoder->videoCodec);
-	if ((encoder->audioCodec && !acodec) || !vcodec || !FFmpegEncoderVerifyContainer(encoder)) {
+	if ((encoder->audioCodec && !acodec) || (encoder->videoCodec && !vcodec) || !FFmpegEncoderVerifyContainer(encoder)) {
+		return false;
+	}
+
+	if (encoder->context) {
 		return false;
 	}
 
@@ -238,8 +248,12 @@ bool FFmpegEncoderOpen(struct FFmpegEncoder* encoder, const char* outfile) {
 			encoder->audio->flags |= CODEC_FLAG_GLOBAL_HEADER;
 #endif
 		}
-		avcodec_open2(encoder->audio, acodec, &opts);
+		int res = avcodec_open2(encoder->audio, acodec, &opts);
 		av_dict_free(&opts);
+		if (res < 0) {
+			FFmpegEncoderClose(encoder);
+			return false;
+		}
 #if LIBAVCODEC_VERSION_MAJOR >= 55
 		encoder->audioFrame = av_frame_alloc();
 #else
@@ -289,6 +303,7 @@ bool FFmpegEncoderOpen(struct FFmpegEncoder* encoder, const char* outfile) {
 #endif
 	}
 
+	if (vcodec) {
 #ifdef FFMPEG_USE_CODECPAR
 		encoder->videoStream = avformat_new_stream(encoder->context, NULL);
 		encoder->video = avcodec_alloc_context3(vcodec);
@@ -339,7 +354,10 @@ bool FFmpegEncoderOpen(struct FFmpegEncoder* encoder, const char* outfile) {
 			encoder->video->pix_fmt = AV_PIX_FMT_YUV444P;
 		}
 
-	avcodec_open2(encoder->video, vcodec, 0);
+		if (avcodec_open2(encoder->video, vcodec, 0) < 0) {
+			FFmpegEncoderClose(encoder);
+			return false;
+		}
 #if LIBAVCODEC_VERSION_MAJOR >= 55
 		encoder->videoFrame = av_frame_alloc();
 #else
@@ -354,35 +372,46 @@ bool FFmpegEncoderOpen(struct FFmpegEncoder* encoder, const char* outfile) {
 #ifdef FFMPEG_USE_CODECPAR
 		avcodec_parameters_from_context(encoder->videoStream->codecpar, encoder->video);
 #endif
+	}
 
-	if (avio_open(&encoder->context->pb, outfile, AVIO_FLAG_WRITE) < 0) {
+	if (avio_open(&encoder->context->pb, outfile, AVIO_FLAG_WRITE) < 0 || avformat_write_header(encoder->context, 0) < 0) {
+		FFmpegEncoderClose(encoder);
 		return false;
 	}
-	return avformat_write_header(encoder->context, 0) >= 0;
+	return true;
 }
 
 void FFmpegEncoderClose(struct FFmpegEncoder* encoder) {
-	if (!encoder->context) {
-		return;
-	}
+	if (encoder->context && encoder->context->pb) {
 		av_write_trailer(encoder->context);
 		avio_close(encoder->context->pb);
+	}
 
-	if (encoder->audioCodec) {
+	if (encoder->postaudioBuffer) {
 		av_free(encoder->postaudioBuffer);
+		encoder->postaudioBuffer = NULL;
+	}
 	if (encoder->audioBuffer) {
 		av_free(encoder->audioBuffer);
+		encoder->audioBuffer = NULL;
 	}
+
+	if (encoder->audioFrame) {
 #if LIBAVCODEC_VERSION_MAJOR >= 55
 		av_frame_free(&encoder->audioFrame);
 #else
 		avcodec_free_frame(&encoder->audioFrame);
 #endif
+	}
+	if (encoder->audio) {
 		avcodec_close(encoder->audio);
+		encoder->audio = NULL;
+	}
 
 	if (encoder->resampleContext) {
 #ifdef USE_LIBAVRESAMPLE
 		avresample_close(encoder->resampleContext);
+		encoder->resampleContext = NULL;
 #else
 		swr_free(&encoder->resampleContext);
 #endif
@@ -393,23 +422,32 @@ void FFmpegEncoderClose(struct FFmpegEncoder* encoder) {
 		av_bsf_free(&encoder->absf);
 #else
 		av_bitstream_filter_close(encoder->absf);
-			encoder->absf = 0;
+		encoder->absf = NULL;
 #endif
 	}
-	}
 
+	if (encoder->videoFrame) {
 #if LIBAVCODEC_VERSION_MAJOR >= 55
 		av_frame_free(&encoder->videoFrame);
 #else
 		avcodec_free_frame(&encoder->videoFrame);
 #endif
-	avcodec_close(encoder->video);
+	}
 
+	if (encoder->video) {
+		avcodec_close(encoder->video);
+		encoder->video = NULL;
+	}
+
+	if (encoder->scaleContext) {
 		sws_freeContext(encoder->scaleContext);
 		encoder->scaleContext = NULL;
+	}
 
+	if (encoder->context) {
 		avformat_free_context(encoder->context);
-	encoder->context = 0;
+		encoder->context = NULL;
+	}
 }
 
 bool FFmpegEncoderIsOpen(struct FFmpegEncoder* encoder) {