From 874533389267c0a83df3b866ac3cc9ce96567ade Mon Sep 17 00:00:00 2001 From: Barry Rowe Date: Fri, 10 May 2019 22:15:39 -0700 Subject: [PATCH] OCR Translation feature finaliziation. (#8738) --- Makefile.common | 6 + command.c | 25 + config.def.h | 4 + configuration.c | 7 + configuration.h | 3 + libretro-common/encodings/encoding_base64.c | 151 ++++++ libretro-common/gfx/scaler/pixconv.c | 22 + libretro-common/gfx/scaler/scaler.c | 2 + libretro-common/include/encodings/base64.h | 48 ++ libretro-common/include/gfx/scaler/pixconv.h | 4 + libretro-common/net/net_http.c | 66 ++- qb/config.libs.sh | 1 + qb/config.params.sh | 1 + translation/translation_service.c | 472 +++++++++++++++++++ translation/translation_service.h | 12 + 15 files changed, 817 insertions(+), 7 deletions(-) create mode 100644 libretro-common/encodings/encoding_base64.c create mode 100644 libretro-common/include/encodings/base64.h create mode 100644 translation/translation_service.c create mode 100644 translation/translation_service.h diff --git a/Makefile.common b/Makefile.common index f0bd252745..91e5db5aba 100644 --- a/Makefile.common +++ b/Makefile.common @@ -180,6 +180,7 @@ OBJ += frontend/frontend.o \ tasks/task_audio_mixer.o \ $(LIBRETRO_COMM_DIR)/encodings/encoding_utf.o \ $(LIBRETRO_COMM_DIR)/encodings/encoding_crc32.o \ + $(LIBRETRO_COMM_DIR)/encodings/encoding_base64.o \ $(LIBRETRO_COMM_DIR)/compat/fopen_utf8.o \ $(LIBRETRO_COMM_DIR)/lists/file_list.o \ $(LIBRETRO_COMM_DIR)/lists/dir_list.o \ @@ -1801,6 +1802,11 @@ ifeq ($(HAVE_NETWORKING), 1) endif endif + ifeq ($(HAVE_TRANSLATE), 1) + DEFINES += -DHAVE_TRANSLATE + OBJ += translation/translation_service.o + endif + ifeq ($(HAVE_NETWORKGAMEPAD), 1) OBJ += input/input_remote.o \ cores/libretro-net-retropad/net_retropad_core.o diff --git a/command.c b/command.c index b969efa20b..18a381bf8d 100755 --- a/command.c +++ b/command.c @@ -50,6 +50,10 @@ #include "discord/discord.h" #endif +#ifdef HAVE_TRANSLATE +#include "translation/translation_service.h" +#endif + #include "midi/midi_driver.h" #ifdef HAVE_MENU @@ -2525,6 +2529,8 @@ TODO: Add a setting for these tweaks */ bool is_idle = false; bool is_slowmotion = false; bool is_perfcnt_enable = false; + settings_t *settings = config_get_ptr(); + #ifdef HAVE_DISCORD discord_userdata_t userdata; #endif @@ -2547,6 +2553,22 @@ TODO: Add a setting for these tweaks */ if (!is_idle) video_driver_cached_frame(); + /* If OCR enabled, translate the screen while paused */ + if (settings->bools.translation_service_enable) + { +#ifdef HAVE_TRANSLATE + if (g_translation_service_status == false) + { + RARCH_LOG("OCR START\n"); + run_translation_service(); + g_translation_service_status = true; + } +#else + RARCH_LOG("OCR Translation not enabled in build. Include HAVE_TRANSLATE define.\n"); +#endif + } + + #ifdef HAVE_DISCORD userdata.status = DISCORD_PRESENCE_GAME_PAUSED; command_event(CMD_EVENT_DISCORD_UPDATE, &userdata); @@ -2556,6 +2578,9 @@ TODO: Add a setting for these tweaks */ { #if defined(HAVE_MENU) && defined(HAVE_MENU_WIDGETS) menu_widgets_set_paused(is_paused); +#endif +#ifdef HAVE_TRANSLATE + g_translation_service_status = false; #endif RARCH_LOG("%s\n", msg_hash_to_str(MSG_UNPAUSED)); command_event(CMD_EVENT_AUDIO_START, NULL); diff --git a/config.def.h b/config.def.h index 5f17be9a0b..703036c0e7 100644 --- a/config.def.h +++ b/config.def.h @@ -450,6 +450,8 @@ static bool menu_swap_ok_cancel_buttons = false; static bool quit_press_twice = false; +static bool default_translation_service_enable = false; + static bool default_log_to_file = false; static bool log_to_file_timestamp = false; @@ -948,4 +950,6 @@ static char buildbot_assets_server_url[] = "http://buildbot.libretro.com/assets/ static char default_discord_app_id[] = "475456035851599874"; +static char default_translation_service_url[] = "http://localhost:4404/"; + #endif diff --git a/configuration.c b/configuration.c index b802b87e17..21057d5a2a 100644 --- a/configuration.c +++ b/configuration.c @@ -1187,6 +1187,8 @@ static struct config_array_setting *populate_settings_array(settings_t *settings SETTING_ARRAY("midi_output", settings->arrays.midi_output, true, midi_output, true); SETTING_ARRAY("youtube_stream_key", settings->arrays.youtube_stream_key, true, NULL, true); SETTING_ARRAY("discord_app_id", settings->arrays.discord_app_id, true, default_discord_app_id, true); + SETTING_ARRAY("translation_service_url", settings->arrays.translation_service_url, true, default_translation_service_url, true); + *size = count; return tmp; @@ -1613,6 +1615,7 @@ static struct config_bool_setting *populate_settings_bool(settings_t *settings, #ifdef HAVE_OZONE SETTING_BOOL("ozone_collapse_sidebar", &settings->bools.ozone_collapse_sidebar, true, ozone_collapse_sidebar, false); #endif + SETTING_BOOL("translation_service_enable", &settings->bools.translation_service_enable, true, default_translation_service_enable, false); SETTING_BOOL("log_to_file", &settings->bools.log_to_file, true, default_log_to_file, false); SETTING_BOOL("log_to_file_timestamp", &settings->bools.log_to_file_timestamp, true, log_to_file_timestamp, false); @@ -1992,6 +1995,10 @@ void config_set_defaults(void) strlcpy(settings->arrays.discord_app_id, default_discord_app_id, sizeof(settings->arrays.discord_app_id)); + strlcpy(settings->arrays.translation_service_url, + default_translation_service_url, sizeof(settings->arrays.translation_service_url)); + + #ifdef HAVE_MATERIALUI if (g_defaults.menu.materialui.menu_color_theme_enable) settings->uints.menu_materialui_color_theme = g_defaults.menu.materialui.menu_color_theme; diff --git a/configuration.h b/configuration.h index 63d9f740fd..d595e10412 100644 --- a/configuration.h +++ b/configuration.h @@ -326,6 +326,8 @@ typedef struct settings bool ozone_collapse_sidebar; + bool translation_service_enable; + bool log_to_file; bool log_to_file_timestamp; @@ -545,6 +547,7 @@ typedef struct settings char twitch_stream_key[PATH_MAX_LENGTH]; char discord_app_id[PATH_MAX_LENGTH]; + char translation_service_url[2048]; } arrays; struct diff --git a/libretro-common/encodings/encoding_base64.c b/libretro-common/encodings/encoding_base64.c new file mode 100644 index 0000000000..7f30cf9e98 --- /dev/null +++ b/libretro-common/encodings/encoding_base64.c @@ -0,0 +1,151 @@ +/* + https://github.com/superwills/NibbleAndAHalf + base64.h -- Fast base64 encoding and decoding. + version 1.0.0, April 17, 2013 143a + Copyright (C) 2013 William Sherif + This software is provided 'as-is', without any express or implied + warranty. In no event will the authors be held liable for any damages + arising from the use of this software. + Permission is granted to anyone to use this software for any purpose, + including commercial applications, and to alter it and redistribute it + freely, subject to the following restrictions: + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + 3. This notice may not be removed or altered from any source distribution. + William Sherif + will.sherif@gmail.com + YWxsIHlvdXIgYmFzZSBhcmUgYmVsb25nIHRvIHVz + + + Modified for RetroArch formatting, logging, and header files. +*/ + + +#include +#include +#include + +/* + Converts binary data of length=len to base64 characters. + Length of the resultant string is stored in flen + (you must pass pointer flen). +*/ +char* base64(const void* binaryData, int len, int *flen) +{ + const unsigned char* bin = (const unsigned char*) binaryData; + char* res; + + int rc = 0; /* result counter */ + int byteNo; /* I need this after the loop */ + + int modulusLen = len % 3 ; + + /* 2 gives 1 and 1 gives 2, but 0 gives 0. */ + int pad = ((modulusLen&1)<<1) + ((modulusLen&2)>>1); + + *flen = 4*(len + pad)/3; + res = (char*) malloc(*flen + 1); /* and one for the null */ + if (!res) + { + /* ERROR: base64 could not allocate enough memory. */ + return 0; + } + + for (byteNo=0; byteNo <= len-3; byteNo+=3) + { + unsigned char BYTE0 = bin[byteNo]; + unsigned char BYTE1 = bin[byteNo+1]; + unsigned char BYTE2 = bin[byteNo+2]; + + res[rc++] = b64[BYTE0 >> 2]; + res[rc++] = b64[((0x3&BYTE0)<<4) + (BYTE1 >> 4)]; + res[rc++] = b64[((0x0f&BYTE1)<<2) + (BYTE2>>6)]; + res[rc++] = b64[0x3f&BYTE2]; + } + + if (pad==2) + { + res[rc++] = b64[bin[byteNo] >> 2]; + res[rc++] = b64[(0x3&bin[byteNo])<<4]; + res[rc++] = '='; + res[rc++] = '='; + } + else if (pad==1) + { + res[rc++] = b64[bin[byteNo] >> 2]; + res[rc++] = b64[((0x3&bin[byteNo])<<4) + (bin[byteNo+1] >> 4)]; + res[rc++] = b64[(0x0f&bin[byteNo+1])<<2]; + res[rc++] = '='; + } + + res[rc]=0; /* NULL TERMINATOR! ;) */ + return res; +} + +unsigned char* unbase64(const char* ascii, int len, int *flen) +{ + const unsigned char *safeAsciiPtr = (const unsigned char*) ascii; + unsigned char *bin; + int cb = 0; + int charNo; + int pad = 0; + + if (len < 2) { /* 2 accesses below would be OOB. */ + /* catch empty string, return NULL as result. */ + + /* ERROR: You passed an invalid base64 string (too short). + * You get NULL back. */ + *flen = 0; + return 0; + } + + if(safeAsciiPtr[len-1]=='=') + ++pad; + if(safeAsciiPtr[len-2]=='=') + ++pad; + + *flen = 3*len/4 - pad; + bin = (unsigned char*)malloc(*flen); + + if (!bin) + { + /* ERROR: unbase64 could not allocate enough memory. */ + return 0; + } + + for (charNo=0; charNo <= len-4-pad; charNo+=4) + { + int A = unb64[safeAsciiPtr[charNo]]; + int B = unb64[safeAsciiPtr[charNo+1]]; + int C = unb64[safeAsciiPtr[charNo+2]]; + int D = unb64[safeAsciiPtr[charNo+3]]; + + bin[cb++] = (A<<2) | (B>>4); + bin[cb++] = (B<<4) | (C>>2); + bin[cb++] = (C<<6) | (D); + } + + if (pad==1) + { + int A = unb64[safeAsciiPtr[charNo]]; + int B = unb64[safeAsciiPtr[charNo+1]]; + int C = unb64[safeAsciiPtr[charNo+2]]; + + bin[cb++] = (A<<2) | (B>>4); + bin[cb++] = (B<<4) | (C>>2); + } + else if (pad==2) + { + int A = unb64[safeAsciiPtr[charNo]]; + int B = unb64[safeAsciiPtr[charNo+1]]; + + bin[cb++] = (A<<2) | (B>>4); + } + + return bin; +} + diff --git a/libretro-common/gfx/scaler/pixconv.c b/libretro-common/gfx/scaler/pixconv.c index 5c6dfe7dd2..66734f9037 100644 --- a/libretro-common/gfx/scaler/pixconv.c +++ b/libretro-common/gfx/scaler/pixconv.c @@ -717,6 +717,28 @@ void conv_bgr24_argb8888(void *output_, const void *input_, } } +void conv_bgr24_rgb565(void *output_, const void *input_, + int width, int height, + int out_stride, int in_stride) +{ + int h, w; + const uint8_t *input = (const uint8_t*)input_; + uint16_t *output = (uint16_t*)output_; + for (h = 0; h < height; + h++, output += out_stride, input += in_stride) + { + const uint8_t *inp = input; + for (w = 0; w < width; w++) + { + uint16_t b = *inp++; + uint16_t g = *inp++; + uint16_t r = *inp++; + + output[w] = ((r & 0x00F8) << 8) | ((g&0x00FC) << 3) | ((b&0x00F8) >> 3); + } + } +} + void conv_argb8888_0rgb1555(void *output_, const void *input_, int width, int height, int out_stride, int in_stride) diff --git a/libretro-common/gfx/scaler/scaler.c b/libretro-common/gfx/scaler/scaler.c index 07a921edce..99805fa496 100644 --- a/libretro-common/gfx/scaler/scaler.c +++ b/libretro-common/gfx/scaler/scaler.c @@ -138,6 +138,8 @@ bool scaler_ctx_gen_filter(struct scaler_ctx *ctx) case SCALER_FMT_ARGB8888: ctx->direct_pixconv = conv_bgr24_argb8888; break; + case SCALER_FMT_RGB565: + ctx->direct_pixconv = conv_bgr24_rgb565; default: break; } diff --git a/libretro-common/include/encodings/base64.h b/libretro-common/include/encodings/base64.h new file mode 100644 index 0000000000..11658d763f --- /dev/null +++ b/libretro-common/include/encodings/base64.h @@ -0,0 +1,48 @@ +#ifndef _LIBRETRO_ENCODINGS_BASE64_H +#define _LIBRETRO_ENCODINGS_BASE64_H + +#include +#include + +#include + +RETRO_BEGIN_DECLS + +const static char* b64 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + +/* maps A=>0,B=>1.. */ +const static unsigned char unb64[]={ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 62, 0, 0, 0, 63, 52, 53, + 54, 55, 56, 57, 58, 59, 60, 61, 0, 0, + 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, + 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, + 25, 0, 0, 0, 0, 0, 0, 26, 27, 28, + 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, + 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, + 49, 50, 51, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, +}; /* This array has 256 elements */ + +char* base64(const void* binaryData, int len, int *flen); +unsigned char* unbase64(const char* ascii, int len, int *flen); + +RETRO_END_DECLS + +#endif diff --git a/libretro-common/include/gfx/scaler/pixconv.h b/libretro-common/include/gfx/scaler/pixconv.h index 7eeba436dc..788f5ee8c5 100644 --- a/libretro-common/include/gfx/scaler/pixconv.h +++ b/libretro-common/include/gfx/scaler/pixconv.h @@ -61,6 +61,10 @@ void conv_bgr24_argb8888(void *output, const void *input, int width, int height, int out_stride, int in_stride); +void conv_bgr24_rgb565(void *output, const void *input, + int width, int height, + int out_stride, int in_stride); + void conv_argb8888_0rgb1555(void *output, const void *input, int width, int height, int out_stride, int in_stride); diff --git a/libretro-common/net/net_http.c b/libretro-common/net/net_http.c index eea5a8c195..9dff75ad06 100644 --- a/libretro-common/net/net_http.c +++ b/libretro-common/net/net_http.c @@ -32,6 +32,7 @@ #endif #include #include +#include #include #include @@ -245,8 +246,16 @@ static void net_http_send_str( struct http_connection_t *net_http_connection_new(const char *url, const char *method, const char *data) { - bool error = false; - char **domain = NULL; + bool error = false; + char **domain = NULL; + char *uri = NULL; + char s[2] = "/"; + char *url_dup = NULL; + char *domain_port = NULL; + char *domain_port2 = NULL; + char *url_port = NULL; + char new_domain[2048]; + struct http_connection_t *conn = (struct http_connection_t*)calloc(1, sizeof(*conn)); @@ -277,6 +286,47 @@ struct http_connection_t *net_http_connection_new(const char *url, else error = true; + /* Get the port here from the url if it's specified. + does not work on username password urls: user:pass@domain.com + + This code is not supposed to be needed, since the port + should be gotten elsewhere when the url is being scanned + for ":", but for whatever reason, it's not working correctly. + */ + + uri = strchr(conn->scan, (char) '/'); + + if (strchr(conn->scan, (char) ':') != NULL) + { + url_dup = strdup(conn->scan); + domain_port = strtok(url_dup, ":"); + domain_port2 = strtok(NULL, ":"); + url_port = domain_port2; + if (strchr(domain_port2, (char) '/') != NULL) + { + url_port = strtok(domain_port2, "/"); + } + + if (url_port != NULL) + { + conn->port = atoi(url_port); + } + + strlcpy(new_domain, domain_port, sizeof(new_domain)); + + if (uri != NULL) + { + if (strchr(uri, (char) '/') == NULL) + strlcat(new_domain, uri, sizeof(new_domain)); + else + { + strlcat(new_domain, "/", sizeof(new_domain)); + strlcat(new_domain, strchr(uri, (char) '/')+sizeof(char), sizeof(new_domain)); + } + strlcpy(conn->scan,new_domain, sizeof(new_domain)); + } + } + /* end of port-fetching from url */ if (error) goto error; @@ -321,13 +371,15 @@ bool net_http_connection_done(struct http_connection_t *conn) if (*conn->scan == '\0') return false; - *conn->scan = '\0'; - if (conn->sock_state.ssl) - conn->port = 443; - else - conn->port = 80; + if (conn->port == 0) + { + if (conn->sock_state.ssl) + conn->port = 443; + else + conn->port = 80; + } if (*conn->scan == ':') { diff --git a/qb/config.libs.sh b/qb/config.libs.sh index 510386e44c..f94af86991 100644 --- a/qb/config.libs.sh +++ b/qb/config.libs.sh @@ -173,6 +173,7 @@ else HAVE_NETWORKGAMEPAD='no' HAVE_CHEEVOS='no' HAVE_DISCORD='no' + HAVE_TRANSLATE='no' HAVE_SSL='no' fi diff --git a/qb/config.params.sh b/qb/config.params.sh index bd62a21888..ef2d7fe4e8 100644 --- a/qb/config.params.sh +++ b/qb/config.params.sh @@ -125,6 +125,7 @@ HAVE_CHEEVOS=yes # Retro Achievements HAVE_LUA=no # Lua support (for Retro Achievements) HAVE_DISCORD=yes # Discord Integration C89_DISCORD=no +HAVE_TRANSLATE=no # OCR and Translation Server Integration HAVE_SHADERPIPELINE=yes # Additional shader-based pipelines C89_SHADERPIPELINE=no HAVE_VULKAN=auto # Vulkan support diff --git a/translation/translation_service.c b/translation/translation_service.c new file mode 100644 index 0000000000..9ad6489058 --- /dev/null +++ b/translation/translation_service.c @@ -0,0 +1,472 @@ +#include +#include +#include +#include +#include +#include "translation_service.h" +#include "gfx/video_driver.h" +#include "gfx/video_frame.h" +#include "gfx/scaler/scaler.h" +#include "tasks/tasks_internal.h" + +#include "configuration.h" +#include "retroarch.h" +#include "verbosity.h" + +bool g_translation_service_status = false; + +static void form_bmp_header(uint8_t *header, unsigned width, unsigned height, + bool is32bpp) +{ + unsigned line_size = (width * (is32bpp?4:3) + 3) & ~3; + unsigned size = line_size * height + 54; + unsigned size_array = line_size * height; + + /* Generic BMP stuff. */ + /* signature */ + header[0] = 'B'; + header[1] = 'M'; + /* file size */ + header[2] = (uint8_t)(size >> 0); + header[3] = (uint8_t)(size >> 8); + header[4] = (uint8_t)(size >> 16); + header[5] = (uint8_t)(size >> 24); + /* reserved */ + header[6] = 0; + header[7] = 0; + header[8] = 0; + header[9] = 0; + /* offset */ + header[10] = 54; + header[11] = 0; + header[12] = 0; + header[13] = 0; + /* DIB size */ + header[14] = 40; + header[15] = 0; + header[16] = 0; + header[17] = 0; + /* Width */ + header[18] = (uint8_t)(width >> 0); + header[19] = (uint8_t)(width >> 8); + header[20] = (uint8_t)(width >> 16); + header[21] = (uint8_t)(width >> 24); + /* Height */ + header[22] = (uint8_t)(height >> 0); + header[23] = (uint8_t)(height >> 8); + header[24] = (uint8_t)(height >> 16); + header[25] = (uint8_t)(height >> 24); + /* Color planes */ + header[26] = 1; + header[27] = 0; + /* Bits per pixel */ + header[28] = 24; + header[29] = 0; + /* Compression method */ + header[30] = 0; + header[31] = 0; + header[32] = 0; + header[33] = 0; + /* Image data size */ + header[34] = (uint8_t)(size_array >> 0); + header[35] = (uint8_t)(size_array >> 8); + header[36] = (uint8_t)(size_array >> 16); + header[37] = (uint8_t)(size_array >> 24); + /* Horizontal resolution */ + header[38] = 19; + header[39] = 11; + header[40] = 0; + header[41] = 0; + /* Vertical resolution */ + header[42] = 19; + header[43] = 11; + header[44] = 0; + header[45] = 0; + /* Palette size */ + header[46] = 0; + header[47] = 0; + header[48] = 0; + header[49] = 0; + /* Important color count */ + header[50] = 0; + header[51] = 0; + header[52] = 0; + header[53] = 0; +} + + +bool run_translation_service(void) +{ + /* + This function does all the stuff needed to translate the game screen, + using the url given in the settings. Once the image from the frame + buffer is sent to the server, the callback will write the translated + image to the screen. + + Supported client/services (thus far) + -Ztranslate client/service ( www.ztranslate.net/docs/service ) + -VGTranslate client ( www.gitlab.com/spherebeaker/vg_translate ) + + To use a client, download the relevant code/release, configure + them, and run them on your local machine, or network. Set the + retroarch configuration to point to your local client (usually + listening on localhost:4404 ) and enable translation service. + + If you don't want to run a client, you can also use a service, + which is basically like someone running a client for you. The + downside here is that your retroarch device will have to have + an internet connection, and you may have to sign up for it. + + + To make your own server, it must listen for a POST request, which + will consist of a json body, with the "image" field as a base64 + encoded string of a 24bit-BMP that the will be translated. The server + must output the translated image in the form of a json body, with + the "image" field also as a base64 encoded, 24bit-BMP. + */ + + size_t pitch; + unsigned width, height; + const void *data = NULL; + uint8_t *bit24_image = NULL; + uint8_t *bit24_image_prev = NULL; + + enum retro_pixel_format pixel_format = video_driver_get_pixel_format(); + struct scaler_ctx *scaler = calloc(1, sizeof(struct scaler_ctx)); + bool error = false; + + uint8_t* bmp_buffer = NULL; + char* bmp64_buffer = NULL; + char* json_buffer = NULL; + + bool retval = false; + struct video_viewport vp; + + uint8_t header[54]; + int out_length = 0; + char* rf1 = "{\"image\": \""; + char* rf2 = "\"}\0"; + + + + if (!scaler) + goto finish; + + video_driver_cached_frame_get(&data, &width, &height, &pitch); + if (!data) + goto finish; + if (data == RETRO_HW_FRAME_BUFFER_VALID) + { + /* + The direct frame capture didn't work, so try getting it + from the viewport instead. This isn't as good as the + raw frame buffer, since the viewport may us bilinear + filtering, or other shaders that will completely trash + the OCR, but it's better than nothing. + */ + vp.x = 0; + vp.y = 0; + vp.width = 0; + vp.height = 0; + vp.full_width = 0; + vp.full_height = 0; + + video_driver_get_viewport_info(&vp); + + if (!vp.width || !vp.height) + goto finish; + + bit24_image_prev = (uint8_t*)malloc(vp.width * vp.height * 3); + bit24_image = (uint8_t*)malloc(width * height * 3); + + if (!bit24_image_prev || !bit24_image) + goto finish; + + if (!video_driver_read_viewport(bit24_image_prev, false)) + goto finish; + + /* Rescale down to regular resolution */ + + + /* + scaler->in_fmt = SCALER_FMT_BGR24; + scaler->in_width = vp.width; + scaler->in_height = vp.height; + + scaler->out_width = width; + scaler->out_height = height; + scaler->out_fmt = SCALER_FMT_BGR24; + + scaler->scaler_type = SCALER_TYPE_POINT; + scaler_ctx_gen_filter(scaler); + + scaler->in_stride = vp.width*3; + scaler->out_stride = width*3; + + scaler_ctx_scale_direct(scaler, bit24_image, bit24_image_prev) + */ + bit24_image = bit24_image_prev; + bit24_image_prev = NULL; + + } + else + { + bit24_image = (uint8_t*) malloc(width*height*3); + if (!bit24_image) + goto finish; + + if (video_driver_get_pixel_format() == RETRO_PIXEL_FORMAT_XRGB8888) + { + scaler->in_fmt = SCALER_FMT_ARGB8888; + RARCH_LOG("IN FORMAT ARGB8888\n"); + } + else + { + scaler->in_fmt = SCALER_FMT_RGB565; + RARCH_LOG("IN FORMAT RGB565\n"); + } + video_frame_convert_to_bgr24( + scaler, + (uint8_t *) bit24_image, + (const uint8_t*)data + ((int)height - 1)*pitch, + width, height, + -pitch); + + scaler_ctx_gen_reset(scaler); + } + + if (!bit24_image) + { + error = true; + goto finish; + } + + /* + at this point, we should have a screenshot in the buffer, so allocate + an array to contain the bmp image along with the bmp header as bytes, + and then covert that to a b64 encoded array for transport in JSON. + */ + form_bmp_header(header, width, height, false); + bmp_buffer = (uint8_t*)malloc(width * height * 3+54); + if (!bmp_buffer) + goto finish; + + memcpy(bmp_buffer, header, 54*sizeof(uint8_t)); + memcpy(bmp_buffer+54, bit24_image, width*height*3*sizeof(uint8_t)); + + bmp64_buffer = base64((void *) bmp_buffer, (int)(width*height*3+54), + &out_length); + if (!bmp64_buffer) + goto finish; + + /* Form request... */ + json_buffer = malloc(11+3+out_length); + if (!json_buffer) + goto finish; + + memcpy(json_buffer, rf1, 11*sizeof(uint8_t)); + memcpy(json_buffer+11, bmp64_buffer, (out_length)*sizeof(uint8_t)); + memcpy(json_buffer+11+out_length, rf2, 3*sizeof(uint8_t)); + + call_translation_server(json_buffer); + error = false; +finish: + if (bit24_image_prev) + free(bit24_image_prev); + if (bit24_image) + free(bit24_image); + + if (scaler) + free(scaler); + + if (bmp_buffer) + free(bmp_buffer); + + if (bmp64_buffer) + free(bmp64_buffer); + + if (json_buffer) + free(json_buffer); + return !error; +} + + +void handle_translation_cb(retro_task_t *task, void *task_data, void *user_data, const char *error) +{ + char* body_copy = NULL; + uint8_t* raw_output_data = NULL; + char* raw_bmp_data = NULL; + struct scaler_ctx* scaler = NULL; + bool is_paused = false; + bool is_idle = false; + bool is_slowmotion = false; + bool is_perfcnt_enable = false; + http_transfer_data_t *data = (http_transfer_data_t*)task_data; + + const char ch = '\"'; + const char s[2] = "\""; + char* ret; + char* string = NULL; + + int new_size = 0; + unsigned width, height; + unsigned image_width, image_height; + size_t pitch; + const void* dummy_data; + void* raw_image_data; + + runloop_get_status(&is_paused, &is_idle, &is_slowmotion, + &is_perfcnt_enable); + + if (!is_paused) + goto finish; + + if (!data || error) + goto finish; + + data->data = (char*)realloc(data->data, data->len + 1); + if (!data->data) + goto finish; + + data->data[data->len] = '\0'; + + /* Parse JSON body for the image data */ + body_copy = strdup(strchr(data->data, ch)); + ret = body_copy; + if (!ret) + goto finish; + + while (strncmp(ret, "\"image\":", strlen("\"image\":"))!=0) + { + ret = strchr(ret, ch); + if (ret == NULL) + break; + else + ret = ret+sizeof(char); + } + + if (ret != NULL) + { + ret = ret + sizeof(char); + ret = ret+strlen("\"image\":")*sizeof(char); + + ret = strchr(ret, ch)+sizeof(char); + string = strtok(ret, s); + } + + if (ret == NULL || string == NULL) + { + error = "Invalid JSON body."; + goto finish; + } + + /* decode the image data from base64 */ + raw_bmp_data = (void*) unbase64(string, strlen(string), &new_size); + if (!raw_bmp_data) + goto finish; + + /* Get the video frame dimensions reference */ + video_driver_cached_frame_get(&dummy_data, &width, &height, &pitch); + + /* Get image data (24 bit), and conver to the emulated pixel format */ + image_width = ((uint32_t) ((uint8_t) raw_bmp_data[21]) << 24) + + ((uint32_t) ((uint8_t) raw_bmp_data[20]) << 16) + + ((uint32_t) ((uint8_t) raw_bmp_data[19]) << 8) + + ((uint32_t) ((uint8_t) raw_bmp_data[18]) << 0); + + image_height = ((uint32_t) ((uint8_t) raw_bmp_data[25]) << 24) + + ((uint32_t) ((uint8_t) raw_bmp_data[24]) << 16) + + ((uint32_t) ((uint8_t) raw_bmp_data[23]) << 8) + + ((uint32_t) ((uint8_t) raw_bmp_data[22]) << 0); + raw_image_data = raw_bmp_data+54*sizeof(uint8_t); + + scaler = calloc(1, sizeof(struct scaler_ctx)); + if (!scaler) + goto finish; + + if (dummy_data == RETRO_HW_FRAME_BUFFER_VALID) + { + /* + In this case, we used the viewport to grab the image + and translate it, and we have the translated image in + the raw_image_data buffer. + */ + + /* TODO: write to the viewport in this case */ + RARCH_LOG("WRITING TO VIEWPORT...\n"); + goto finish; + } + + /* The assigned pitch may not be reliable. The width of + the video frame can change during run-time, but the + pitch may not, so we just assign it as the width + times the byte depth. + */ + + if (video_driver_get_pixel_format() == RETRO_PIXEL_FORMAT_XRGB8888) + { + raw_output_data = (uint8_t*) malloc(width*height*4*sizeof(uint8_t)); + scaler->out_fmt = SCALER_FMT_ARGB8888; + pitch = width*4; + scaler->out_stride = width*4; + } + else + { + raw_output_data = (uint8_t*) malloc(width*height*2*sizeof(uint8_t)); + scaler->out_fmt = SCALER_FMT_RGB565; + pitch = width*2; + scaler->out_stride = width*1; + } + + if (!raw_output_data) + goto finish; + scaler->in_fmt = SCALER_FMT_BGR24; + scaler->in_width = image_width; + scaler->in_height = image_height; + scaler->out_width = width; + scaler->out_height = height; + scaler->scaler_type = SCALER_TYPE_POINT; + scaler_ctx_gen_filter(scaler); + scaler->in_stride = -1*width*3; + + scaler_ctx_scale_direct(scaler, raw_output_data, (uint8_t*)raw_image_data+(image_height-1)*width*3); + /* + video_driver_frame(raw_output_data+(height-1)*pitch, image_width, image_height, -pitch); + */ + video_driver_frame(raw_output_data, image_width, image_height, pitch); + RARCH_LOG("Translation done.\n"); +finish: + if (error) + RARCH_ERR("%s: %s\n", msg_hash_to_str(MSG_DOWNLOAD_FAILED), error); + + if (data) + { + if (data->data) + free(data->data); + free(data); + } + if (user_data) + free(user_data); + + if (body_copy) + free(body_copy); + + if (raw_bmp_data) + free(raw_bmp_data); + + if (scaler) + free(scaler); + + if (raw_output_data) + free(raw_output_data); +} + + +void call_translation_server(const char* body) +{ + settings_t *settings = config_get_ptr(); + + RARCH_LOG("Server url: %s\n", settings->arrays.translation_service_url); + task_push_http_post_transfer(settings->arrays.translation_service_url, + body, true, NULL, handle_translation_cb, NULL); +} diff --git a/translation/translation_service.h b/translation/translation_service.h new file mode 100644 index 0000000000..ec9135bde3 --- /dev/null +++ b/translation/translation_service.h @@ -0,0 +1,12 @@ +#ifndef __TRANSLATION_SERVICE__H +#define __TRANSLATION_SERVICE__H + +#include "tasks/tasks_internal.h" +void call_translation_server(const char* body); + +bool g_translation_service_status; + +bool run_translation_service(void); + +void handle_translation_cb(retro_task_t *task, void *task_data, void *user_data, const char *error); +#endif