From 79d2e95d02c3e68d2420bf75ace07f739a383a03 Mon Sep 17 00:00:00 2001 From: Vicki Pfau Date: Fri, 24 Jul 2020 23:58:48 -0700 Subject: [PATCH 01/12] GB: Fix OAM in GB mVL playback --- src/gb/extra/proxy.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gb/extra/proxy.c b/src/gb/extra/proxy.c index 5b5dbc3bd..f7b21cbda 100644 --- a/src/gb/extra/proxy.c +++ b/src/gb/extra/proxy.c @@ -128,7 +128,7 @@ static bool _parsePacket(struct mVideoLogger* logger, const struct mVideoLoggerD break; case DIRTY_OAM: if (item->address < GB_SIZE_OAM) { - logger->oam[item->address] = item->value; + ((uint8_t*) logger->oam)[item->address] = item->value; proxyRenderer->backend->writeOAM(proxyRenderer->backend, item->address); } break; From 3795a64b775107bac77e03bbd2601402de57e043 Mon Sep 17 00:00:00 2001 From: Vicki Pfau Date: Sat, 25 Jul 2020 23:37:06 -0700 Subject: [PATCH 02/12] Test: I added strlcpy for a reason --- src/platform/test/cinema-main.c | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/src/platform/test/cinema-main.c b/src/platform/test/cinema-main.c index 2a9206f65..d2f105305 100644 --- a/src/platform/test/cinema-main.c +++ b/src/platform/test/cinema-main.c @@ -11,6 +11,7 @@ #include #include +#include #include #include #include @@ -127,7 +128,7 @@ static bool parseCInemaArgs(int argc, char* const* argv) { } break; case 'b': - strncpy(base, optarg, sizeof(base)); + strlcpy(base, optarg, sizeof(base)); // TODO: Verify path exists break; case 'd': @@ -140,7 +141,7 @@ static bool parseCInemaArgs(int argc, char* const* argv) { dryRun = true; break; case 'o': - strncpy(outdir, optarg, sizeof(outdir)); + strlcpy(outdir, optarg, sizeof(outdir)); // TODO: Make directory break; case 'q': @@ -244,7 +245,7 @@ static void reduceTestList(struct CInemaTestList* tests) { } static void testToPath(const char* testName, char* path) { - strncpy(path, base, PATH_MAX); + strlcpy(path, base, PATH_MAX); bool dotSeen = true; size_t i; @@ -253,7 +254,7 @@ static void testToPath(const char* testName, char* path) { dotSeen = true; } else { if (dotSeen) { - strncpy(&path[i], PATH_SEP, PATH_MAX - i); + strlcpy(&path[i], PATH_SEP, PATH_MAX - i); i += strlen(PATH_SEP); dotSeen = false; if (!i) { @@ -268,7 +269,7 @@ static void testToPath(const char* testName, char* path) { static void _loadConfigTree(struct Table* configTree, const char* testName) { char key[MAX_TEST]; - strncpy(key, testName, sizeof(key) - 1); + strlcpy(key, testName, sizeof(key)); struct mCoreConfig* config; while (!(config = HashTableLookup(configTree, key))) { @@ -301,7 +302,7 @@ static const char* _lookupValue(struct Table* configTree, const char* testName, _loadConfigTree(configTree, testName); char testKey[MAX_TEST]; - strncpy(testKey, testName, sizeof(testKey) - 1); + strlcpy(testKey, testName, sizeof(testKey)); struct mCoreConfig* config; while (true) { @@ -378,10 +379,10 @@ bool CInemaTestInit(struct CInemaTest* test, const char* directory, const char* return false; } memset(test, 0, sizeof(*test)); - strncpy(test->directory, directory, sizeof(test->directory) - 1); - strncpy(test->filename, filename, sizeof(test->filename) - 1); + strlcpy(test->directory, directory, sizeof(test->directory)); + strlcpy(test->filename, filename, sizeof(test->filename)); directory += strlen(base) + 1; - strncpy(test->name, directory, sizeof(test->name) - 1); + strlcpy(test->name, directory, sizeof(test->name)); char* str = strstr(test->name, PATH_SEP); while (str) { str[0] = '.'; @@ -446,7 +447,7 @@ static bool _loadBaseline(struct VDir* dir, struct CInemaImage* image, size_t fr static struct VDir* _makeOutDir(const char* testName) { char path[PATH_MAX] = {0}; - strncpy(path, outdir, sizeof(path) - 1); + strlcpy(path, outdir, sizeof(path)); char* pathEnd = path + strlen(path); const char* pos; while (true) { @@ -768,7 +769,7 @@ int main(int argc, char** argv) { #ifndef _WIN32 char* rbase = realpath(base, NULL); if (rbase) { - strncpy(base, rbase, PATH_MAX); + strlcpy(base, rbase, sizeof(base)); free(rbase); } #endif From 390ed6c83ccf761e86a97c656ac6dfad649e5439 Mon Sep 17 00:00:00 2001 From: Vicki Pfau Date: Sun, 28 Jun 2020 19:03:21 -0700 Subject: [PATCH 03/12] FFmpeg: Decoder skeleton --- CMakeLists.txt | 4 +- src/feature/ffmpeg/ffmpeg-common.h | 37 ++++++ src/feature/ffmpeg/ffmpeg-decoder.c | 170 ++++++++++++++++++++++++++++ src/feature/ffmpeg/ffmpeg-decoder.h | 43 +++++++ src/feature/ffmpeg/ffmpeg-encoder.c | 1 + src/feature/ffmpeg/ffmpeg-encoder.h | 26 +---- src/platform/test/cinema-main.c | 14 ++- 7 files changed, 269 insertions(+), 26 deletions(-) create mode 100644 src/feature/ffmpeg/ffmpeg-common.h create mode 100644 src/feature/ffmpeg/ffmpeg-decoder.c create mode 100644 src/feature/ffmpeg/ffmpeg-decoder.h diff --git a/CMakeLists.txt b/CMakeLists.txt index ea7c3905f..ce569b95c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -535,8 +535,8 @@ if(USE_FFMPEG) endif() include_directories(AFTER ${FFMPEG_INCLUDE_DIRS} ${LIBAVCODEC_INCLUDE_DIRS} ${LIBAVFILTER_INCLUDE_DIRS} ${LIBAVFORMAT_INCLUDE_DIRS} ${LIBAVRESAMPLE_INCLUDE_DIRS} ${LIBAVUTIL_INCLUDE_DIRS} ${LIBSWRESAMPLE_INCLUDE_DIRS} ${LIBSWSCALE_INCLUDE_DIRS}) link_directories(${FFMPEG_LIBRARY_DIRS} ${LIBAVCODEC_LIBRARY_DIRS} ${LIBAVFILTER_LIBRARY_DIRS} ${LIBAVFORMAT_LIBRARY_DIRS} ${LIBAVRESAMPLE_LIBRARY_DIRS} ${LIBAVUTIL_LIBRARY_DIRS} ${LIBSWRESAMPLE_LIBRARY_DIRS} ${LIBSWSCALE_LIBRARY_DIRS}) - list(APPEND FEATURE_SRC "${CMAKE_CURRENT_SOURCE_DIR}/src/feature/ffmpeg/ffmpeg-encoder.c") - list(APPEND DEPENDENCY_LIB ${FFMPEG_LIBRARIES} ${LIBAVCODEC_LIBRARIES} ${LIBAVFILTER_LIBRARIES} ${LIBAVFORMAT_LIBRARIES} ${LIBAVRESAMPLE_LIBRARIES} ${LIBAVUTIL_LIBRARIES} ${LIBSWSCALE_LIBRARIES} ${LIBSWRESAMPLE_LIBRARIES}) + list(APPEND FEATURE_SRC "${CMAKE_CURRENT_SOURCE_DIR}/src/feature/ffmpeg/ffmpeg-encoder.c" "${CMAKE_CURRENT_SOURCE_DIR}/src/feature/ffmpeg/ffmpeg-decoder.c") + list(APPEND DEPENDENCY_LIB ${FFMPEG_LIBRARIES} ${LIBAVCODEC_LIBRARIES} ${LIBAVFILTER_LIBRARIES} ${LIBAVFORMAT_LIBRARIES} ${LIBAVRESAMPLE_LIBRARIES} ${LIBAVUTIL_LIBRARIES} ${LIBSWSCALE_LIBRARIES} ${LIBSWRESAMPLE_LIBRARIES}) if(WIN32 AND NOT DEFINED VCPKG_TARGET_TRIPLET) list(APPEND DEPENDENCY_LIB bcrypt) endif() diff --git a/src/feature/ffmpeg/ffmpeg-common.h b/src/feature/ffmpeg/ffmpeg-common.h new file mode 100644 index 000000000..d006c83ce --- /dev/null +++ b/src/feature/ffmpeg/ffmpeg-common.h @@ -0,0 +1,37 @@ +/* Copyright (c) 2013-2020 Jeffrey Pfau + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +#ifndef FFMPEG_COMMON +#define FFMPEG_COMMON + +#include + +CXX_GUARD_START + +#include +#include + +// Version 57.16 in FFmpeg +#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(57, 37, 100) +#define FFMPEG_USE_PACKETS +#endif + +// Version 57.15 in libav +#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(57, 35, 0) +#define FFMPEG_USE_NEW_BSF +#endif + +// Version 57.14 in libav +#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(57, 48, 0) +#define FFMPEG_USE_CODECPAR +#endif + +#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(57, 8, 0) +#define FFMPEG_USE_PACKET_UNREF +#endif + +CXX_GUARD_END + +#endif diff --git a/src/feature/ffmpeg/ffmpeg-decoder.c b/src/feature/ffmpeg/ffmpeg-decoder.c new file mode 100644 index 000000000..d9c73b267 --- /dev/null +++ b/src/feature/ffmpeg/ffmpeg-decoder.c @@ -0,0 +1,170 @@ +/* Copyright (c) 2013-2020 Jeffrey Pfau + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +#include "ffmpeg-decoder.h" + +void FFmpegDecoderInit(struct FFmpegDecoder* decoder) { +#if LIBAVFORMAT_VERSION_INT < AV_VERSION_INT(58, 9, 100) + av_register_all(); +#endif + + memset(decoder, 0, sizeof(*decoder)); + decoder->audioStream = -1; + decoder->videoStream = -1; +} + +bool FFmpegDecoderOpen(struct FFmpegDecoder* decoder, const char* infile) { + if (FFmpegDecoderIsOpen(decoder)) { + return false; + } + + if (avformat_open_input(&decoder->context, infile, NULL, NULL) < 0) { + return false; + } + + if (avformat_find_stream_info(decoder->context, NULL) < 0) { + FFmpegDecoderClose(decoder); + return false; + } + + unsigned i; + for (i = 0; i < decoder->context->nb_streams; ++i) { +#ifdef FFMPEG_USE_CODECPAR + enum AVMediaType type = decoder->context->streams[i]->codecpar->codec_type; +#else + enum AVMediaType type = decoder->context->streams[i]->codec->codec_type; +#endif + struct AVCodec* codec; + struct AVCodecContext* context = NULL; + if (type == AVMEDIA_TYPE_VIDEO && decoder->videoStream < 0) { + decoder->video = avcodec_alloc_context3(NULL); + if (!decoder->video) { + FFmpegDecoderClose(decoder); + return false; + } + context = decoder->video; + } + + if (type == AVMEDIA_TYPE_AUDIO && decoder->audioStream < 0) { + decoder->audio = avcodec_alloc_context3(NULL); + if (!decoder->audio) { + FFmpegDecoderClose(decoder); + return false; + } + context = decoder->audio; + } + if (!context) { + continue; + } + +#ifdef FFMPEG_USE_CODECPAR + if (avcodec_parameters_to_context(context, decoder->context->streams[i]->codecpar) < 0) { + FFmpegDecoderClose(decoder); + return false; + } +#endif + codec = avcodec_find_decoder(context->codec_id); + if (!codec) { + FFmpegDecoderClose(decoder); + return false; + } + if (avcodec_open2(context, codec, NULL) < 0) { + FFmpegDecoderClose(decoder); + return false; + } + + if (type == AVMEDIA_TYPE_VIDEO) { + decoder->videoStream = i; + decoder->width = context->coded_width; + decoder->height = context->coded_height; + if (decoder->d.videoDimensionsChanged) { + decoder->d.videoDimensionsChanged(&decoder->d, decoder->width, decoder->height); + } +#if LIBAVCODEC_VERSION_MAJOR >= 55 + decoder->videoFrame = av_frame_alloc(); +#else + decoder->videoFrame = avcodec_alloc_frame(); +#endif + } + + if (type == AVMEDIA_TYPE_AUDIO) { + decoder->audioStream = i; +#if LIBAVCODEC_VERSION_MAJOR >= 55 + decoder->audioFrame = av_frame_alloc(); +#else + decoder->audioFrame = avcodec_alloc_frame(); +#endif + } + } + return true; +} + +void FFmpegDecoderClose(struct FFmpegDecoder* decoder) { + if (decoder->audioFrame) { +#if LIBAVCODEC_VERSION_MAJOR >= 55 + av_frame_free(&decoder->audioFrame); +#else + avcodec_free_frame(&decoder->audioFrame); +#endif + } + + if (decoder->audio) { + avcodec_close(decoder->audio); + decoder->audio = NULL; + } + + if (decoder->videoFrame) { +#if LIBAVCODEC_VERSION_MAJOR >= 55 + av_frame_free(&decoder->videoFrame); +#else + avcodec_free_frame(&decoder->videoFrame); +#endif + } + + if (decoder->video) { + avcodec_close(decoder->video); + decoder->video = NULL; + } + + if (decoder->context) { + avformat_close_input(&decoder->context); + } +} + +bool FFmpegDecoderIsOpen(struct FFmpegDecoder* decoder) { + return !!decoder->context; +} + +bool FFmpegDecoderRead(struct FFmpegDecoder* decoder) { + while (true) { + AVPacket packet; + if (av_read_frame(decoder->context, &packet) < 0) { + return false; + } + + if (packet.stream_index == decoder->audioStream) { + + } else if (packet.stream_index == decoder->videoStream) { +#ifdef FFMPEG_USE_CODECPAR + if (avcodec_send_packet(decoder->video, &packet) < 0) { + + } + if (avcodec_receive_frame(decoder->video, decoder->videoFrame) < 0) { + + } +#else + int gotData; + if (avcodec_decode_video2(decoder->video, decoder->videoFrame, &gotData, &packet) < 0 || !gotData) { + + } +#endif + } +#ifdef FFMPEG_USE_PACKET_UNREF + av_packet_unref(&packet); +#else + av_free_packet(&packet); +#endif + } +} \ No newline at end of file diff --git a/src/feature/ffmpeg/ffmpeg-decoder.h b/src/feature/ffmpeg/ffmpeg-decoder.h new file mode 100644 index 000000000..b382115bb --- /dev/null +++ b/src/feature/ffmpeg/ffmpeg-decoder.h @@ -0,0 +1,43 @@ +/* Copyright (c) 2013-2020 Jeffrey Pfau + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +#ifndef FFMPEG_DECODER +#define FFMPEG_DECODER + +#include + +CXX_GUARD_START + +#include + +#include "feature/ffmpeg/ffmpeg-common.h" + +#define FFMPEG_DECODER_BUFSIZE 4096 + +struct FFmpegDecoder { + struct mAVStream d; + struct AVFormatContext* context; + + int audioStream; + AVFrame* audioFrame; + struct AVCodecContext* audio; + + int videoStream; + AVFrame* videoFrame; + struct AVCodecContext* video; + + int width; + int height; +}; + +void FFmpegDecoderInit(struct FFmpegDecoder*); +bool FFmpegDecoderOpen(struct FFmpegDecoder*, const char* infile); +void FFmpegDecoderClose(struct FFmpegDecoder*); +bool FFmpegDecoderIsOpen(struct FFmpegDecoder*); +bool FFmpegDecoderRead(struct FFmpegDecoder*); + +CXX_GUARD_END + +#endif diff --git a/src/feature/ffmpeg/ffmpeg-encoder.c b/src/feature/ffmpeg/ffmpeg-encoder.c index 1e3da858b..71145f696 100644 --- a/src/feature/ffmpeg/ffmpeg-encoder.c +++ b/src/feature/ffmpeg/ffmpeg-encoder.c @@ -7,6 +7,7 @@ #include #include +#include #include #include diff --git a/src/feature/ffmpeg/ffmpeg-encoder.h b/src/feature/ffmpeg/ffmpeg-encoder.h index 746e8c46e..562d71edc 100644 --- a/src/feature/ffmpeg/ffmpeg-encoder.h +++ b/src/feature/ffmpeg/ffmpeg-encoder.h @@ -10,29 +10,9 @@ CXX_GUARD_START -#include +#include -#include -#include - -// Version 57.16 in FFmpeg -#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(57, 37, 100) -#define FFMPEG_USE_PACKETS -#endif - -// Version 57.15 in libav -#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(57, 35, 0) -#define FFMPEG_USE_NEW_BSF -#endif - -// Version 57.14 in libav -#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(57, 48, 0) -#define FFMPEG_USE_CODECPAR -#endif - -#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(57, 8, 0) -#define FFMPEG_USE_PACKET_UNREF -#endif +#include "feature/ffmpeg/ffmpeg-common.h" #define FFMPEG_FILTERS_MAX 4 @@ -73,7 +53,7 @@ struct FFmpegEncoder { struct AVCodecContext* video; enum AVPixelFormat pixFormat; enum AVPixelFormat ipixFormat; - struct AVFrame* videoFrame; + AVFrame* videoFrame; int width; int height; int iwidth; diff --git a/src/platform/test/cinema-main.c b/src/platform/test/cinema-main.c index d2f105305..501014ff6 100644 --- a/src/platform/test/cinema-main.c +++ b/src/platform/test/cinema-main.c @@ -16,6 +16,10 @@ #include #include +#ifdef USE_FFMPEG +#include "feature/ffmpeg/ffmpeg-decoder.h" +#endif + #ifdef _MSC_VER #include #else @@ -566,10 +570,12 @@ void CInemaTestRun(struct CInemaTest* test, struct Table* configTree) { unsigned limit = 9999; unsigned skip = 0; unsigned fail = 0; + unsigned video = 0; CInemaConfigGetUInt(configTree, test->name, "frames", &limit); CInemaConfigGetUInt(configTree, test->name, "skip", &skip); CInemaConfigGetUInt(configTree, test->name, "fail", &fail); + CInemaConfigGetUInt(configTree, test->name, "video", &video); CInemaConfigLoad(configTree, test->name, core); core->loadROM(core, rom); @@ -600,7 +606,13 @@ void CInemaTestRun(struct CInemaTest* test, struct Table* configTree) { .height = image.height, .stride = image.width, }; - if (_loadBaseline(dir, &expected, frame, &test->status)) { + bool baselineFound = false; + if (video) { + baselineFound = false; + } else { + baselineFound = _loadBaseline(dir, &expected, frame, &test->status); + } + if (baselineFound) { uint8_t* testPixels = image.data; uint8_t* expectPixels = expected.data; size_t x; From 051fd94b70ae010e977b1daf4b1491f978e2f9e7 Mon Sep 17 00:00:00 2001 From: Vicki Pfau Date: Fri, 10 Jul 2020 01:10:32 -0700 Subject: [PATCH 04/12] FFmpeg: Get frame decoding working --- src/feature/ffmpeg/ffmpeg-decoder.c | 50 ++++++++++++++++++++++++----- src/feature/ffmpeg/ffmpeg-decoder.h | 4 ++- 2 files changed, 45 insertions(+), 9 deletions(-) diff --git a/src/feature/ffmpeg/ffmpeg-decoder.c b/src/feature/ffmpeg/ffmpeg-decoder.c index d9c73b267..d3c2b726e 100644 --- a/src/feature/ffmpeg/ffmpeg-decoder.c +++ b/src/feature/ffmpeg/ffmpeg-decoder.c @@ -5,6 +5,8 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ #include "ffmpeg-decoder.h" +#include + void FFmpegDecoderInit(struct FFmpegDecoder* decoder) { #if LIBAVFORMAT_VERSION_INT < AV_VERSION_INT(58, 9, 100) av_register_all(); @@ -79,14 +81,15 @@ bool FFmpegDecoderOpen(struct FFmpegDecoder* decoder, const char* infile) { decoder->videoStream = i; decoder->width = context->coded_width; decoder->height = context->coded_height; - if (decoder->d.videoDimensionsChanged) { - decoder->d.videoDimensionsChanged(&decoder->d, decoder->width, decoder->height); + if (decoder->out->videoDimensionsChanged) { + decoder->out->videoDimensionsChanged(decoder->out, decoder->width, decoder->height); } #if LIBAVCODEC_VERSION_MAJOR >= 55 decoder->videoFrame = av_frame_alloc(); #else decoder->videoFrame = avcodec_alloc_frame(); #endif + decoder->pixels = malloc(decoder->width * decoder->height * BYTES_PER_PIXEL); } if (type == AVMEDIA_TYPE_AUDIO) { @@ -111,8 +114,17 @@ void FFmpegDecoderClose(struct FFmpegDecoder* decoder) { } if (decoder->audio) { +#ifdef FFMPEG_USE_CODECPAR + avcodec_free_context(&decoder->audio); +#else avcodec_close(decoder->audio); decoder->audio = NULL; +#endif + } + + if (decoder->scaleContext) { + sws_freeContext(decoder->scaleContext); + decoder->scaleContext = NULL; } if (decoder->videoFrame) { @@ -123,9 +135,18 @@ void FFmpegDecoderClose(struct FFmpegDecoder* decoder) { #endif } + if (decoder->pixels) { + free(decoder->pixels); + decoder->pixels = NULL; + } + if (decoder->video) { +#ifdef FFMPEG_USE_CODECPAR + avcodec_free_context(&decoder->video); +#else avcodec_close(decoder->video); decoder->video = NULL; +#endif } if (decoder->context) { @@ -138,28 +159,40 @@ bool FFmpegDecoderIsOpen(struct FFmpegDecoder* decoder) { } bool FFmpegDecoderRead(struct FFmpegDecoder* decoder) { - while (true) { + bool readPacket = false; + while (!readPacket) { AVPacket packet; if (av_read_frame(decoder->context, &packet) < 0) { - return false; + break; } + readPacket = true; if (packet.stream_index == decoder->audioStream) { - + // TODO } else if (packet.stream_index == decoder->videoStream) { #ifdef FFMPEG_USE_CODECPAR if (avcodec_send_packet(decoder->video, &packet) < 0) { - + // TODO } if (avcodec_receive_frame(decoder->video, decoder->videoFrame) < 0) { - + readPacket = false; } #else int gotData; if (avcodec_decode_video2(decoder->video, decoder->videoFrame, &gotData, &packet) < 0 || !gotData) { - + readPacket = false; } #endif + if (readPacket && decoder->out->postVideoFrame) { + if (!decoder->scaleContext) { + decoder->scaleContext = sws_getContext(decoder->width, decoder->height, decoder->videoFrame->format, + decoder->width, decoder->height, AV_PIX_FMT_BGR32, + SWS_POINT, 0, 0, 0); + } + int stride = decoder->width * BYTES_PER_PIXEL; + sws_scale(decoder->scaleContext, (const uint8_t* const*) decoder->videoFrame->data, decoder->videoFrame->linesize, 0, decoder->videoFrame->height, &decoder->pixels, &stride); + decoder->out->postVideoFrame(decoder->out, (const color_t*) decoder->pixels, decoder->width); + } } #ifdef FFMPEG_USE_PACKET_UNREF av_packet_unref(&packet); @@ -167,4 +200,5 @@ bool FFmpegDecoderRead(struct FFmpegDecoder* decoder) { av_free_packet(&packet); #endif } + return readPacket; } \ No newline at end of file diff --git a/src/feature/ffmpeg/ffmpeg-decoder.h b/src/feature/ffmpeg/ffmpeg-decoder.h index b382115bb..d2b8ebab6 100644 --- a/src/feature/ffmpeg/ffmpeg-decoder.h +++ b/src/feature/ffmpeg/ffmpeg-decoder.h @@ -17,7 +17,7 @@ CXX_GUARD_START #define FFMPEG_DECODER_BUFSIZE 4096 struct FFmpegDecoder { - struct mAVStream d; + struct mAVStream* out; struct AVFormatContext* context; int audioStream; @@ -27,9 +27,11 @@ struct FFmpegDecoder { int videoStream; AVFrame* videoFrame; struct AVCodecContext* video; + struct SwsContext* scaleContext; int width; int height; + uint8_t* pixels; }; void FFmpegDecoderInit(struct FFmpegDecoder*); From 1b755e17d062cef0e94c1e509e58a62e3a94d03a Mon Sep 17 00:00:00 2001 From: Vicki Pfau Date: Fri, 10 Jul 2020 01:23:50 -0700 Subject: [PATCH 05/12] Test: CInema supports videos now --- src/platform/test/cinema-main.c | 104 ++++++++++++++++++++++++++++++-- 1 file changed, 100 insertions(+), 4 deletions(-) diff --git a/src/platform/test/cinema-main.c b/src/platform/test/cinema-main.c index 501014ff6..340723986 100644 --- a/src/platform/test/cinema-main.c +++ b/src/platform/test/cinema-main.c @@ -18,6 +18,7 @@ #ifdef USE_FFMPEG #include "feature/ffmpeg/ffmpeg-decoder.h" +#include "feature/ffmpeg/ffmpeg-encoder.h" #endif #ifdef _MSC_VER @@ -395,7 +396,7 @@ bool CInemaTestInit(struct CInemaTest* test, const char* directory, const char* return true; } -static bool _loadBaseline(struct VDir* dir, struct CInemaImage* image, size_t frame, enum CInemaStatus* status) { +static bool _loadBaselinePNG(struct VDir* dir, struct CInemaImage* image, size_t frame, enum CInemaStatus* status) { char baselineName[32]; snprintf(baselineName, sizeof(baselineName), "baseline_%04" PRIz "u.png", frame); struct VFile* baselineVF = dir->openFile(dir, baselineName, O_RDONLY); @@ -449,6 +450,32 @@ static bool _loadBaseline(struct VDir* dir, struct CInemaImage* image, size_t fr return true; } +#ifdef USE_FFMPEG +struct CInemaStream { + struct mAVStream d; + struct CInemaImage* image; + enum CInemaStatus* status; +}; + +static void _cinemaDimensionsChanged(struct mAVStream* stream, unsigned width, unsigned height) { + struct CInemaStream* cistream = (struct CInemaStream*) stream; + if (height != cistream->image->height || width != cistream->image->width) { + CIerr(1, "Size mismatch for video, expected %ux%u, got %ux%u\n", width, height, cistream->image->width, cistream->image->height); + if (*cistream->status == CI_PASS) { + *cistream->status = CI_FAIL; + } + } +} + +static void _cinemaVideoFrame(struct mAVStream* stream, const color_t* pixels, size_t stride) { + struct CInemaStream* cistream = (struct CInemaStream*) stream; + cistream->image->stride = stride; + size_t bufferSize = cistream->image->stride * cistream->image->height * BYTES_PER_PIXEL; + cistream->image->data = malloc(bufferSize); + memcpy(cistream->image->data, pixels, bufferSize); +} +#endif + static struct VDir* _makeOutDir(const char* testName) { char path[PATH_MAX] = {0}; strlcpy(path, outdir, sizeof(path)); @@ -590,6 +617,45 @@ void CInemaTestRun(struct CInemaTest* test, struct Table* configTree) { for (frame = 0; frame < skip; ++frame) { core->runFrame(core); } + +#ifdef USE_FFMPEG + struct FFmpegDecoder decoder; + struct FFmpegEncoder encoder; + struct CInemaStream stream = {0}; + if (video) { + char fname[PATH_MAX]; + snprintf(fname, sizeof(fname), "%s" PATH_SEP "baseline.mkv", test->directory); + if (rebaseline) { + FFmpegEncoderInit(&encoder); + FFmpegEncoderSetAudio(&encoder, NULL, 0); + FFmpegEncoderSetVideo(&encoder, "png", 0, 0); + FFmpegEncoderSetContainer(&encoder, "mkv"); + FFmpegEncoderSetDimensions(&encoder, image.width, image.height); + if (!FFmpegEncoderOpen(&encoder, fname)) { + CIerr(1, "Failed to save baseline video\n"); + } else { + core->setAVStream(core, &encoder.d); + } + } else { + FFmpegDecoderInit(&decoder); + stream.d.postVideoFrame = _cinemaVideoFrame; + stream.d.videoDimensionsChanged = _cinemaDimensionsChanged; + stream.status = &test->status; + decoder.out = &stream.d; + stream.image = ℑ + + if (!FFmpegDecoderOpen(&decoder, fname)) { + CIerr(1, "Failed to load baseline video\n"); + } + } + } +#else + if (video) { + CIerr(0, "Failed to run video test without ffmpeg linked in\n"); + test->status = CI_ERROR; + } +#endif + for (frame = 0; limit; ++frame, --limit) { core->runFrame(core); ++test->totalFrames; @@ -606,11 +672,24 @@ void CInemaTestRun(struct CInemaTest* test, struct Table* configTree) { .height = image.height, .stride = image.width, }; - bool baselineFound = false; + bool baselineFound; if (video) { baselineFound = false; +#ifdef USE_FFMPEG + if (!rebaseline && FFmpegDecoderIsOpen(&decoder)) { + stream.image = &expected; + while (!expected.data) { + if (!FFmpegDecoderRead(&decoder)) { + CIerr(1, "Failed to read more frames. EOF?\n"); + test->status = CI_FAIL; + break; + } + } + baselineFound = expected.data; + } +#endif } else { - baselineFound = _loadBaseline(dir, &expected, frame, &test->status); + baselineFound = _loadBaselinePNG(dir, &expected, frame, &test->status); } if (baselineFound) { uint8_t* testPixels = image.data; @@ -707,8 +786,10 @@ void CInemaTestRun(struct CInemaTest* test, struct Table* configTree) { free(expected.data); } else if (test->status == CI_ERROR) { break; - } else if (rebaseline) { + } else if (rebaseline && !video) { _writeBaseline(dir, &image, frame); + } else if (!rebaseline) { + test->status = CI_FAIL; } } @@ -720,6 +801,16 @@ void CInemaTestRun(struct CInemaTest* test, struct Table* configTree) { } } +#ifdef USE_FFMPEG + if (video) { + if (rebaseline) { + FFmpegEncoderClose(&encoder); + } else { + FFmpegDecoderClose(&decoder); + } + } +#endif + free(image.data); mCoreConfigDeinit(&core->config); core->deinit(core); @@ -791,6 +882,11 @@ int main(int argc, char** argv) { struct mLogger logger = { .log = _log }; mLogSetDefaultLogger(&logger); +#ifdef USE_FFMPEG + if (verbosity < 2) { + av_log_set_level(AV_LOG_ERROR); + } +#endif if (argc > 0) { size_t i; From db4f1ecb2deb6dcc0c17057770f84240f7fe60be Mon Sep 17 00:00:00 2001 From: Vicki Pfau Date: Fri, 10 Jul 2020 23:53:55 -0700 Subject: [PATCH 06/12] FFmpeg: Minor lossless encoding improvements --- src/feature/ffmpeg/ffmpeg-encoder.c | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/src/feature/ffmpeg/ffmpeg-encoder.c b/src/feature/ffmpeg/ffmpeg-encoder.c index 71145f696..fc2896d41 100644 --- a/src/feature/ffmpeg/ffmpeg-encoder.c +++ b/src/feature/ffmpeg/ffmpeg-encoder.c @@ -390,12 +390,18 @@ bool FFmpegEncoderOpen(struct FFmpegEncoder* encoder, const char* outfile) { // QuickTime and a few other things require YUV420 encoder->video->pix_fmt = AV_PIX_FMT_YUV420P; } -#if LIBAVCODEC_VERSION_MAJOR >= 57 if (encoder->video->codec->id == AV_CODEC_ID_FFV1) { +#if LIBAVCODEC_VERSION_MAJOR >= 57 av_opt_set(encoder->video->priv_data, "coder", "range_tab", 0); - } + av_opt_set_int(encoder->video->priv_data, "context", 1, 0); #endif + encoder->video->gop_size = 128; + encoder->video->level = 3; + } + if (encoder->video->codec->id == AV_CODEC_ID_PNG) { + encoder->video->compression_level = 8; + } if (strcmp(vcodec->name, "libx264") == 0) { // Try to adaptively figure out when you can use a slower encoder if (encoder->width * encoder->height > 1000000) { @@ -406,13 +412,15 @@ bool FFmpegEncoderOpen(struct FFmpegEncoder* encoder, const char* outfile) { av_opt_set(encoder->video->priv_data, "preset", "faster", 0); } if (encoder->videoBitrate == 0) { - av_opt_set(encoder->video->priv_data, "crf", "0", 0); + av_opt_set(encoder->video->priv_data, "qp", "0", 0); encoder->video->pix_fmt = AV_PIX_FMT_YUV444P; } } if (strcmp(vcodec->name, "libvpx-vp9") == 0 && encoder->videoBitrate == 0) { - av_opt_set(encoder->video->priv_data, "lossless", "1", 0); - encoder->video->pix_fmt = AV_PIX_FMT_YUV444P; + av_opt_set_int(encoder->video->priv_data, "lossless", 1, 0); + av_opt_set_int(encoder->video->priv_data, "crf", 0, 0); + encoder->video->gop_size = 120; + encoder->video->pix_fmt = AV_PIX_FMT_GBRP; } if (strcmp(vcodec->name, "libwebp_anim") == 0 && encoder->videoBitrate == 0) { av_opt_set(encoder->video->priv_data, "lossless", "1", 0); From faceb902c823411f8199705a02e196e8cf969337 Mon Sep 17 00:00:00 2001 From: Vicki Pfau Date: Sat, 11 Jul 2020 13:44:23 -0700 Subject: [PATCH 07/12] Test: Fix non-SGB video tests --- src/platform/test/cinema-main.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/platform/test/cinema-main.c b/src/platform/test/cinema-main.c index 340723986..d6a86967e 100644 --- a/src/platform/test/cinema-main.c +++ b/src/platform/test/cinema-main.c @@ -617,6 +617,7 @@ void CInemaTestRun(struct CInemaTest* test, struct Table* configTree) { for (frame = 0; frame < skip; ++frame) { core->runFrame(core); } + core->desiredVideoDimensions(core, &image.width, &image.height); #ifdef USE_FFMPEG struct FFmpegDecoder decoder; From 481f0f0b0ea2fc0e27c2aa97e148d3cad10bd59e Mon Sep 17 00:00:00 2001 From: Vicki Pfau Date: Sat, 11 Jul 2020 13:45:08 -0700 Subject: [PATCH 08/12] Test: Fix writing diffs --- src/platform/test/cinema-main.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/platform/test/cinema-main.c b/src/platform/test/cinema-main.c index d6a86967e..e92026854 100644 --- a/src/platform/test/cinema-main.c +++ b/src/platform/test/cinema-main.c @@ -715,7 +715,7 @@ void CInemaTestRun(struct CInemaTest* test, struct Table* configTree) { if (r | g | b) { failed = true; if (diffs && !diff) { - diff = calloc(expected.width * expected.height, BYTES_PER_PIXEL); + diff = calloc(expected.stride * expected.height, BYTES_PER_PIXEL); } CIlog(3, "Frame %u failed at pixel %" PRIz "ux%" PRIz "u with diff %i,%i,%i (expected %02x%02x%02x, got %02x%02x%02x)\n", frameCounter, x, y, r, g, b, @@ -763,9 +763,9 @@ void CInemaTestRun(struct CInemaTest* test, struct Table* configTree) { if (failed) { struct CInemaImage outdiff = { .data = diff, - .width = image.width, - .height = image.height, - .stride = image.width, + .width = expected.width, + .height = expected.height, + .stride = expected.stride, }; _writeDiff(test->name, &image, frame, "result"); From b83fba5b3da66767dedb28120f319f0178ebd7cb Mon Sep 17 00:00:00 2001 From: Vicki Pfau Date: Sat, 11 Jul 2020 22:04:47 -0700 Subject: [PATCH 09/12] Test: Move much of the logging to stdout --- src/platform/test/cinema-main.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/platform/test/cinema-main.c b/src/platform/test/cinema-main.c index e92026854..0d0a36726 100644 --- a/src/platform/test/cinema-main.c +++ b/src/platform/test/cinema-main.c @@ -460,7 +460,7 @@ struct CInemaStream { static void _cinemaDimensionsChanged(struct mAVStream* stream, unsigned width, unsigned height) { struct CInemaStream* cistream = (struct CInemaStream*) stream; if (height != cistream->image->height || width != cistream->image->width) { - CIerr(1, "Size mismatch for video, expected %ux%u, got %ux%u\n", width, height, cistream->image->width, cistream->image->height); + CIlog(1, "Size mismatch for video, expected %ux%u, got %ux%u\n", width, height, cistream->image->width, cistream->image->height); if (*cistream->status == CI_PASS) { *cistream->status = CI_FAIL; } @@ -681,7 +681,7 @@ void CInemaTestRun(struct CInemaTest* test, struct Table* configTree) { stream.image = &expected; while (!expected.data) { if (!FFmpegDecoderRead(&decoder)) { - CIerr(1, "Failed to read more frames. EOF?\n"); + CIlog(1, "Failed to read more frames. EOF?\n"); test->status = CI_FAIL; break; } From f5ed6a6c814f506b61b4d0ce4827eb555f1059fc Mon Sep 17 00:00:00 2001 From: Vicki Pfau Date: Mon, 13 Jul 2020 18:51:18 -0700 Subject: [PATCH 10/12] FFmpeg: Fix decoder dimensions changing --- src/feature/ffmpeg/ffmpeg-decoder.c | 45 +++++++++++++++++++---------- src/platform/test/cinema-main.c | 1 - 2 files changed, 30 insertions(+), 16 deletions(-) diff --git a/src/feature/ffmpeg/ffmpeg-decoder.c b/src/feature/ffmpeg/ffmpeg-decoder.c index d3c2b726e..5d3c3aa08 100644 --- a/src/feature/ffmpeg/ffmpeg-decoder.c +++ b/src/feature/ffmpeg/ffmpeg-decoder.c @@ -79,17 +79,13 @@ bool FFmpegDecoderOpen(struct FFmpegDecoder* decoder, const char* infile) { if (type == AVMEDIA_TYPE_VIDEO) { decoder->videoStream = i; - decoder->width = context->coded_width; - decoder->height = context->coded_height; - if (decoder->out->videoDimensionsChanged) { - decoder->out->videoDimensionsChanged(decoder->out, decoder->width, decoder->height); - } + decoder->width = -1; + decoder->height = -1; #if LIBAVCODEC_VERSION_MAJOR >= 55 decoder->videoFrame = av_frame_alloc(); #else decoder->videoFrame = avcodec_alloc_frame(); #endif - decoder->pixels = malloc(decoder->width * decoder->height * BYTES_PER_PIXEL); } if (type == AVMEDIA_TYPE_AUDIO) { @@ -161,7 +157,9 @@ bool FFmpegDecoderIsOpen(struct FFmpegDecoder* decoder) { bool FFmpegDecoderRead(struct FFmpegDecoder* decoder) { bool readPacket = false; while (!readPacket) { - AVPacket packet; + AVPacket packet = { + .stream_index = -2 + }; if (av_read_frame(decoder->context, &packet) < 0) { break; } @@ -183,15 +181,32 @@ bool FFmpegDecoderRead(struct FFmpegDecoder* decoder) { readPacket = false; } #endif - if (readPacket && decoder->out->postVideoFrame) { - if (!decoder->scaleContext) { - decoder->scaleContext = sws_getContext(decoder->width, decoder->height, decoder->videoFrame->format, - decoder->width, decoder->height, AV_PIX_FMT_BGR32, - SWS_POINT, 0, 0, 0); + if (readPacket) { + if (decoder->width != decoder->videoFrame->width || decoder->height != decoder->videoFrame->height) { + decoder->width = decoder->videoFrame->width; + decoder->height = decoder->videoFrame->height; + if (decoder->out->videoDimensionsChanged) { + decoder->out->videoDimensionsChanged(decoder->out, decoder->width, decoder->height); + } + if (decoder->pixels) { + free(decoder->pixels); + } + decoder->pixels = calloc(decoder->width * decoder->height, BYTES_PER_PIXEL); + if (decoder->scaleContext) { + sws_freeContext(decoder->scaleContext); + decoder->scaleContext = NULL; + } + } + if (decoder->out->postVideoFrame) { + if (!decoder->scaleContext) { + decoder->scaleContext = sws_getContext(decoder->width, decoder->height, decoder->videoFrame->format, + decoder->width, decoder->height, AV_PIX_FMT_BGR32, + SWS_POINT, 0, 0, 0); + } + int stride = decoder->width * BYTES_PER_PIXEL; + sws_scale(decoder->scaleContext, (const uint8_t* const*) decoder->videoFrame->data, decoder->videoFrame->linesize, 0, decoder->videoFrame->height, &decoder->pixels, &stride); + decoder->out->postVideoFrame(decoder->out, (const color_t*) decoder->pixels, decoder->width); } - int stride = decoder->width * BYTES_PER_PIXEL; - sws_scale(decoder->scaleContext, (const uint8_t* const*) decoder->videoFrame->data, decoder->videoFrame->linesize, 0, decoder->videoFrame->height, &decoder->pixels, &stride); - decoder->out->postVideoFrame(decoder->out, (const color_t*) decoder->pixels, decoder->width); } } #ifdef FFMPEG_USE_PACKET_UNREF diff --git a/src/platform/test/cinema-main.c b/src/platform/test/cinema-main.c index 0d0a36726..fb1003585 100644 --- a/src/platform/test/cinema-main.c +++ b/src/platform/test/cinema-main.c @@ -643,7 +643,6 @@ void CInemaTestRun(struct CInemaTest* test, struct Table* configTree) { stream.d.videoDimensionsChanged = _cinemaDimensionsChanged; stream.status = &test->status; decoder.out = &stream.d; - stream.image = ℑ if (!FFmpegDecoderOpen(&decoder, fname)) { CIerr(1, "Failed to load baseline video\n"); From 7ecdf94b6beb5c5a78d59b9822067bb253368595 Mon Sep 17 00:00:00 2001 From: Vicki Pfau Date: Wed, 22 Jul 2020 22:20:01 -0700 Subject: [PATCH 11/12] Test: Load dummy save in CInema --- src/platform/test/cinema-main.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/platform/test/cinema-main.c b/src/platform/test/cinema-main.c index fb1003585..6d40ea41d 100644 --- a/src/platform/test/cinema-main.c +++ b/src/platform/test/cinema-main.c @@ -605,7 +605,11 @@ void CInemaTestRun(struct CInemaTest* test, struct Table* configTree) { CInemaConfigGetUInt(configTree, test->name, "video", &video); CInemaConfigLoad(configTree, test->name, core); + struct VFile* save = VFileMemChunk(NULL, 0); core->loadROM(core, rom); + if (!core->loadSave(core, save)) { + save->close(save); + } core->rtc.override = RTC_FAKE_EPOCH; core->rtc.value = 1200000000; core->reset(core); From c7035f6c347423dc0a13ca5e72a002448758f87e Mon Sep 17 00:00:00 2001 From: Vicki Pfau Date: Thu, 23 Jul 2020 19:37:54 -0700 Subject: [PATCH 12/12] FFmpeg: Add ZMBV specialization --- src/feature/ffmpeg/ffmpeg-encoder.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/feature/ffmpeg/ffmpeg-encoder.c b/src/feature/ffmpeg/ffmpeg-encoder.c index fc2896d41..9752971d4 100644 --- a/src/feature/ffmpeg/ffmpeg-encoder.c +++ b/src/feature/ffmpeg/ffmpeg-encoder.c @@ -402,6 +402,12 @@ bool FFmpegEncoderOpen(struct FFmpegEncoder* encoder, const char* outfile) { if (encoder->video->codec->id == AV_CODEC_ID_PNG) { encoder->video->compression_level = 8; } +#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(58, 48, 100) + if (encoder->video->codec->id == AV_CODEC_ID_ZMBV) { + encoder->video->compression_level = 5; + encoder->video->pix_fmt = AV_PIX_FMT_BGR0; + } +#endif if (strcmp(vcodec->name, "libx264") == 0) { // Try to adaptively figure out when you can use a slower encoder if (encoder->width * encoder->height > 1000000) {