Xunkar's AI service rework updated (#15640)

* AI service rework

* File missing

* Fixed C89 build

* Fixed usage of inline for C89 build

* Fixed an overlay unloading bug

Made sure to unload the overlay on release and when the server returns empty values in automatic modes.

* Fixed forward decl (c89)

* Fixed OpenGL texture loading

Moved image display to the main thread for now

* Changed some formatting slightly

* Fixed struct variable order and put brackets on newlines

* Moved pointer, fixed retroarch.cfg, and replaced strlcat with strlcpy

* Fixed catenation issue

* Fixed a few other catenation issues

* Fixed one more concatenation  issue

* Fixed concatenation issue

* Fixed a few other concatenation issues

* Fixed one more concatenation  issue

* potential fix for parsing issue

---------

Co-authored-by: Xunkar <329857+xunkar@users.noreply.github.com>

accessibility.h

@@ -31,11 +31,36 @@
 #endif
 
 #include "configuration.h"
+#include "tasks/tasks_internal.h"
+
+#ifdef HAVE_THREADS
+#include "rthreads/rthreads.h"
+#endif
 
 typedef struct
 {
+   /* The last request task, used to prepare and send the translation */
+   retro_task_t *request_task;
+   
+   /* The last response task, used to parse costly translation data */
+   retro_task_t *response_task;
+   
+   /* Timestamp of the last translation request */
+   retro_time_t last_call;
+
+   #ifdef HAVE_THREADS
+   /* Necessary because last_image is manipulated by task handlers */
+   slock_t *image_lock;
+   #endif
+
+   /* Frame captured during the last call to the translation service */
+   uint8_t *last_image;
+   int last_image_size;
+
+   /* 1 if the automatic mode has been enabled, 0 otherwise */
    int ai_service_auto;
-   /* Is text-to-speech accessibility turned on? */
+   
+   /* Text-to-speech narrator override flag */
    bool enabled;
 } access_state_t;
 
@@ -46,42 +71,73 @@ bool is_narrator_running(bool accessibility_enable);
 #endif
 
 /*
-   This function does all the stuff needed to translate the game screen,
+   Invoke this method to send a request to the AI service. 
-   using the URL given in the settings.  Once the image from the frame
+   It makes the following POST request using URL params:
-   buffer is sent to the server, the callback will write the translated
+      – source_lang (optional): language code of the content currently running.
-   image to the screen.
+      – target_lang (optional): language of the content to return.
+      – output: comma-separated list of formats that must be provided by the
+         service. Also lists supported sub-formats.
          
-   Supported client/services (thus far)
+   The currently supported formats are:
-   -VGTranslate client ( www.gitlab.com/spherebeaker/vg_translate )
+      – sound: raw audio to playback. (wav)
-   -Ztranslate client/service ( www.ztranslate.net/docs/service )
+      – text: text to be read through internal text-to-speech capabilities.
+         'subs' can be specified on top of that to explain that we are looking
+         for short text response in the manner of subtitles.
+      – image: image to display on top of the video feed. Widgets will be used
+         first if possible, otherwise we'll try to draw it directly on the 
+         video buffer. (bmp, png, png-a) [All in 24-bits BGR formats]
          
-   To use a client, download the relevant code/release, configure
+   In addition, the request contains a JSON payload, formatted as such:
-   them, and run them on your local machine, or network.  Set the
+      – image: captured frame from the currently running content (in base64).
-   retroarch configuration to point to your local client (usually
+      – format: format of the captured frame ("png", or "bmp").
-   listening on localhost:4404 ) and enable translation service.
+      – coords: array describing the coordinates of the image within the 
+         viewport space (x, y, width, height).
+      – viewport: array describing the size of the viewport (width, height).
+      – label: a text string describing the content (<system id>__<content id>).
+      – state: a JSON object describing the state of the frontend, containing:
+         – paused: 1 if the content has been paused, 0 otherwise.
+         – <key>: the name of a retropad input, valued 1 if pressed.
+            (a, b, x, y, l, r, l2, r2, l3, r3)
+            (up, down, left, right, start, select)
             
-   If you don't want to run a client, you can also use a service,
+   The translation component then expects a response from the AI service in the
-   which is basically like someone running a client for you.  The
+   form of a JSON payload, formatted as such:
-   downside here is that your retroarch device will have to have
+      – image: base64 representation of an image in a supported format.
-   an internet connection, and you may have to sign up for it.
+      – sound: base64 representation of a sound byte in a supported format.
+      – text: results from the service as a string.
+      – text_position: hint for the position of the text when the service is
+         running in text mode (ie subtitles). Position is a number,
+         1 for Bottom or 2 for Top (defaults to bottom).
+      – press: a list of retropad input to forcibly press. On top of the 
+         expected keys (cf. 'state' above) values 'pause' and 'unpause' can be
+         specified to control the flow of the content.
+      – error: any error encountered with the request.
+      – auto: either 'auto' or 'continue' to control automatic requests.
       
-   To make your own server, it must listen for a POST request, which
+   All fields are optional, but at least one of them must be present.
-   will consist of a JSON body, with the "image" field as a base64
+   If 'error' is set, the error is shown to the user and everything else is
-   encoded string of a 24bit-BMP/PNG that the will be translated.
+   ignored, even 'auto' settings.
-   The server must output the translated image in the form of a
-   JSON body, with the "image" field also as a base64 encoded
-   24bit-BMP, or as an alpha channel png.
    
-  "paused" boolean is passed in to indicate if the current call
+   With 'auto' on 'auto', RetroArch will automatically send a new request
-   was made during a paused frame.  Due to how the menu widgets work,
+   (with a minimum delay enforced by uints.ai_service_poll_delay), with a value
-   if the ai service is called in "auto" mode, then this call will
+   of 'continue', RetroArch will ignore the returned content and skip to the 
-   be made while the menu widgets unpause the core for a frame to update
+   next automatic request. This allows the service to specify that the returned
-   the on-screen widgets.  To tell the ai service what the pause
+   content is the same as the one previously sent, so RetroArch does not need to
-   mode is honestly, we store the runloop_paused variable from before
+   update its display unless necessary. With 'continue' the service *must* 
-   the handle_translation_cb wipes the widgets, and pass that in here.
+   still send the content, as we may need to display it if the user paused the 
+   AI service for instance.
+
+   {paused} boolean is passed in to indicate if the current call was made 
+   during a paused frame. Due to how the menu widgets work, if the AI service 
+   is called in 'auto' mode, then this call will be made while the menu widgets 
+   unpause the core for a frame to update the on-screen widgets. To tell the AI
+   service what the pause mode is honestly, we store the runloop_paused 
+   variable from before the service wipes the widgets, and pass that in here.
 */
 bool run_translation_service(settings_t *settings, bool paused);
 
+void translation_release(bool inform);
+
 bool accessibility_speak_priority(
       bool accessibility_enable,
       unsigned accessibility_narrator_speech_speed,

config.def.h

@@ -1749,8 +1749,14 @@
 
 #define DEFAULT_AI_SERVICE_MODE 1
 
+#define DEFAULT_AI_SERVICE_TEXT_POSITION 0
+#define DEFAULT_AI_SERVICE_TEXT_PADDING 5
+
 #define DEFAULT_AI_SERVICE_URL "http://localhost:4404/"
 
+#define DEFAULT_AI_SERVICE_POLL_DELAY 0
+#define MAXIMUM_AI_SERVICE_POLL_DELAY 500
+
 #if defined(HAVE_FFMPEG) || defined(HAVE_MPV)
 #define DEFAULT_BUILTIN_MEDIAPLAYER_ENABLE true
 #else

configuration.c

@@ -2477,11 +2477,13 @@ static struct config_uint_setting *populate_settings_uint(
    SETTING_UINT("cheevos_appearance_anchor",     &settings->uints.cheevos_appearance_anchor, true, DEFAULT_CHEEVOS_APPEARANCE_ANCHOR, false);
    SETTING_UINT("cheevos_visibility_summary",    &settings->uints.cheevos_visibility_summary, true, DEFAULT_CHEEVOS_VISIBILITY_SUMMARY, false);
 #endif
-
    SETTING_UINT("accessibility_narrator_speech_speed", &settings->uints.accessibility_narrator_speech_speed, true, DEFAULT_ACCESSIBILITY_NARRATOR_SPEECH_SPEED, false);
    SETTING_UINT("ai_service_mode",              &settings->uints.ai_service_mode,            true, DEFAULT_AI_SERVICE_MODE, false);
    SETTING_UINT("ai_service_target_lang",       &settings->uints.ai_service_target_lang,     true, 0, false);
    SETTING_UINT("ai_service_source_lang",       &settings->uints.ai_service_source_lang,     true, 0, false);
+   SETTING_UINT("ai_service_poll_delay",        &settings->uints.ai_service_poll_delay,      true, DEFAULT_AI_SERVICE_POLL_DELAY, false);
+   SETTING_UINT("ai_service_text_position",     &settings->uints.ai_service_text_position,   true, DEFAULT_AI_SERVICE_TEXT_POSITION, false);
+   SETTING_UINT("ai_service_text_padding",      &settings->uints.ai_service_text_padding,    true, DEFAULT_AI_SERVICE_TEXT_PADDING, false);
 
 #ifdef HAVE_LIBNX
    SETTING_UINT("libnx_overclock",               &settings->uints.libnx_overclock, true, SWITCH_DEFAULT_CPU_PROFILE, false);

configuration.h

@@ -334,6 +334,9 @@ typedef struct settings
       unsigned ai_service_mode;
       unsigned ai_service_target_lang;
       unsigned ai_service_source_lang;
+      unsigned ai_service_poll_delay;
+      unsigned ai_service_text_position;
+      unsigned ai_service_text_padding;
 
       unsigned core_updater_auto_backup_history_size;
       unsigned video_black_frame_insertion;

frontend/drivers/platform_win32.c

@@ -1064,9 +1064,12 @@ static bool accessibility_speak_windows(int speed,
       if (!wc || res != 0) 
       {
          RARCH_ERR("Error communicating with NVDA\n");
+         /* Fallback on powershell immediately and retry */
+         g_plat_win32_flags &= ~PLAT_WIN32_FLAG_USE_NVDA;
+         g_plat_win32_flags |= PLAT_WIN32_FLAG_USE_POWERSHELL;
          if (wc)
             free(wc);
-         return false;
+         return accessibility_speak_windows(speed, speak_text, priority);
       }
 
       nvdaController_cancelSpeech_func();

gfx/gfx_widgets.c

@@ -1471,6 +1471,67 @@ static void INLINE gfx_widgets_font_unbind(gfx_widget_font_data_t *font_data)
    font_driver_bind_block(font_data->font, NULL);
 }
 
+#ifdef HAVE_TRANSLATE
+static void gfx_widgets_ai_line(
+      video_frame_info_t *video, char *line, int line_idx, int line_total)
+{
+   settings_t *settings       = config_get_ptr();
+   gfx_display_t *p_disp      = (gfx_display_t*)video->disp_userdata;
+   dispgfx_widget_t *p_widget = (dispgfx_widget_t*)video->widgets_userdata;
+   void *userdata             = video->userdata;
+   unsigned video_width       = video->width;
+   unsigned video_height      = video->height;
+   
+   int line_width             = font_driver_get_message_width(
+         p_widget->gfx_widget_fonts.regular.font,
+         line, strlen(line), 1.0f);
+   
+   int hpadding               = p_widget->simple_widget_padding;
+   int vpadding               = settings->uints.ai_service_text_padding;
+   int half_vw                = video_width * 0.5f;
+   int block_width            = line_width + hpadding * 2;
+   int block_height           = p_widget->simple_widget_height;
+   int block_x                = half_vw - block_width * 0.5f;
+   int block_y                = 0;
+   int line_y                 = 0;
+   
+   int position               = (settings->uints.ai_service_text_position > 0)
+         ? settings->uints.ai_service_text_position
+         : p_widget->ai_service_text_position;
+   
+   switch (position)
+   {
+      case 0: /* Undef. */
+      case 1: /* Bottom */
+         block_y  = (video_height * (100 - vpadding) * 0.01f)
+                  - ((line_total - line_idx) * block_height);
+         break;
+      case 2: /* Top    */
+         block_y  = (video_height * (vpadding * 0.01f))
+                  + (line_idx * block_height);
+         break;
+   }
+   
+   line_y = block_y + block_height * 0.5f 
+          + p_widget->gfx_widget_fonts.regular.line_centre_offset;
+  
+   gfx_display_set_alpha(p_widget->backdrop_orig, DEFAULT_BACKDROP);
+   
+   gfx_display_draw_quad(
+         p_disp, userdata, video_width, video_height,
+         block_x, block_y, block_width, block_height,
+         video_width, video_height,
+         p_widget->backdrop_orig,
+         NULL);
+   
+   gfx_widgets_draw_text(
+         &p_widget->gfx_widget_fonts.regular,
+         line, half_vw, line_y,
+         video_width, video_height,
+         0xFFFFFFFF, TEXT_ALIGN_CENTER, true);
+}
+#endif
+
 void gfx_widgets_frame(void *data)
 {
    size_t i;
@@ -1520,12 +1581,8 @@ void gfx_widgets_frame(void *data)
    /* AI Service overlay */
    if (p_dispwidget->ai_service_overlay_state > 0)
    {
-      float outline_color[16] = {
+      int text_length = strlen(p_dispwidget->ai_service_text);
-      0.00, 1.00, 0.00, 1.00,
+      
-      0.00, 1.00, 0.00, 1.00,
-      0.00, 1.00, 0.00, 1.00,
-      0.00, 1.00, 0.00, 1.00,
-      };
       gfx_display_set_alpha(p_dispwidget->pure_white, 1.0f);
 
       if (p_dispwidget->ai_service_overlay_texture)
@@ -1551,62 +1608,45 @@ void gfx_widgets_frame(void *data)
             dispctx->blend_end(userdata);
       }
       
-      /* top line */
+      /* AI Service subtitle overlay widget */
-      gfx_display_draw_quad(
+      if (text_length > 0)
-            p_disp,
+      {
-            userdata,
+         int padding      = p_dispwidget->simple_widget_padding;
-            video_width, video_height,
+         int text_width   = font_driver_get_message_width(
-            0, 0,
+               p_dispwidget->gfx_widget_fonts.regular.font,
-            video_width,
+               p_dispwidget->ai_service_text,
-            p_dispwidget->divider_width_1px,
+               text_length, 1.0f);
-            video_width,
+         
-            video_height,
+         if (text_width > (video_width * 0.9f - padding * 2))
-            outline_color,
+         {
-            NULL
+            int text_half     = text_length / 2;
-            );
+            char *extra_line  = (char*)malloc(sizeof(char) * text_length);
-      /* bottom line */
+            for (; text_half > 0; text_half--)
-      gfx_display_draw_quad(
+            {
-            p_disp,
+               if (p_dispwidget->ai_service_text[text_half] == ' ')
-            userdata,
+               {
-            video_width, video_height,
+                  p_dispwidget->ai_service_text[text_half] = '\0';
-            0,
+                  gfx_widgets_ai_line(
-            video_height - p_dispwidget->divider_width_1px,
+                        video_info, p_dispwidget->ai_service_text, 0, 2);
-            video_width,
+                  strlcpy(
-            p_dispwidget->divider_width_1px,
+                        extra_line, 
-            video_width,
+                        p_dispwidget->ai_service_text + text_half + 1,
-            video_height,
+                        text_length - text_half);
-            outline_color,
+                  gfx_widgets_ai_line(
-            NULL
+                        video_info, extra_line, 1, 2);
-            );
+                        
-      /* left line */
+                  p_dispwidget->ai_service_text[text_half] = ' ';
-      gfx_display_draw_quad(
+                  free(extra_line);
-            p_disp,
+                  break;
-            userdata,
+               }
-            video_width,
+            }
-            video_height,
+         } 
-            0,
+         else 
-            0,
+         {
-            p_dispwidget->divider_width_1px,
+            gfx_widgets_ai_line(
-            video_height,
+                  video_info, p_dispwidget->ai_service_text, 0, 1);
-            video_width,
+         }
-            video_height,
+      }
-            outline_color,
-            NULL
-            );
-      /* right line */
-      gfx_display_draw_quad(
-            p_disp,
-            userdata,
-            video_width, video_height,
-            video_width - p_dispwidget->divider_width_1px,
-            0,
-            p_dispwidget->divider_width_1px,
-            video_height,
-            video_width,
-            video_height,
-            outline_color,
-            NULL
-            );
 
       if (p_dispwidget->ai_service_overlay_state == 2)
           p_dispwidget->ai_service_overlay_state = 3;
@@ -2149,6 +2189,7 @@ void gfx_widgets_ai_service_overlay_unload(void)
    if (p_dispwidget->ai_service_overlay_state == 1)
    {
       video_driver_texture_unload(&p_dispwidget->ai_service_overlay_texture);
+      p_dispwidget->ai_service_text[0]         = '\0';
       p_dispwidget->ai_service_overlay_texture = 0;
       p_dispwidget->ai_service_overlay_state   = 0;
    }

gfx/gfx_widgets.h

@@ -236,6 +236,8 @@ typedef struct dispgfx_widget
 #ifdef HAVE_TRANSLATE
    unsigned ai_service_overlay_width;
    unsigned ai_service_overlay_height;
+   unsigned ai_service_text_position;
+   char ai_service_text[255];
 #endif
 
    uint8_t flags;

intl/msg_hash_lbl.h

@@ -6031,6 +6031,18 @@ MSG_HASH(
    MENU_ENUM_LABEL_AI_SERVICE_SOURCE_LANG,
    "ai_service_source_lang"
    )
+MSG_HASH(
+   MENU_ENUM_LABEL_AI_SERVICE_POLL_DELAY,
+   "ai_service_poll_delay"
+   )
+MSG_HASH(
+   MENU_ENUM_LABEL_AI_SERVICE_TEXT_POSITION,
+   "ai_service_text_position"
+   )
+MSG_HASH(
+   MENU_ENUM_LABEL_AI_SERVICE_TEXT_PADDING,
+   "ai_service_text_padding"
+   )
 MSG_HASH(
    MENU_ENUM_LABEL_SETTINGS_SHOW_DRIVERS,
    "settings_show_drivers"

intl/msg_hash_us.h

@@ -6565,9 +6565,9 @@ MSG_HASH(
    MENU_ENUM_LABEL_VALUE_AI_SERVICE_MODE,
    "AI Service Output"
    )
-MSG_HASH( /* FIXME What does the Narrator mode do? */
+MSG_HASH(
    MENU_ENUM_SUBLABEL_AI_SERVICE_MODE,
-   "Show translation as a text overlay (Image Mode), or play as Text-To-Speech (Speech Mode)."
+   "Show translation as an image overlay (Image Mode), as direct audio (Speech), text-to-speech (Narrator), or text overlay (Text)."
    )
 MSG_HASH(
    MENU_ENUM_LABEL_VALUE_AI_SERVICE_URL,
@@ -6609,6 +6609,30 @@ MSG_HASH(
    MENU_ENUM_SUBLABEL_AI_SERVICE_TARGET_LANG,
    "The language the service will translate to. 'Default' is English."
    )
+MSG_HASH(
+   MENU_ENUM_LABEL_VALUE_AI_SERVICE_POLL_DELAY,
+   "AI Service Auto-Polling Delay"
+   )
+MSG_HASH(
+   MENU_ENUM_SUBLABEL_AI_SERVICE_POLL_DELAY,
+   "Minimum delay in ms between automatic calls. Lowers reactivity but increases CPU performance."
+   )
+MSG_HASH(
+   MENU_ENUM_LABEL_VALUE_AI_SERVICE_TEXT_POSITION,
+   "AI Service Text Position Override"
+   )
+MSG_HASH(
+   MENU_ENUM_SUBLABEL_AI_SERVICE_TEXT_POSITION,
+   "Override for the position of the overlay, when the service is in Text mode."
+   )
+MSG_HASH(
+   MENU_ENUM_LABEL_VALUE_AI_SERVICE_TEXT_PADDING,
+   "AI Service Text Padding (%)"
+   )
+MSG_HASH(
+   MENU_ENUM_SUBLABEL_AI_SERVICE_TEXT_PADDING,
+   "Vertical padding to apply to the text overlay, when the service is in Text mode. More padding will push the text towards the center of the screen."
+   )
 
 /* Settings > Accessibility */
 
@@ -10176,6 +10200,26 @@ MSG_HASH(
    MENU_ENUM_LABEL_VALUE_AI_SERVICE_NARRATOR_MODE,
    "Narrator Mode"
    )
+MSG_HASH(
+   MENU_ENUM_LABEL_VALUE_AI_SERVICE_TEXT_MODE,
+   "Text Mode"
+   )
+MSG_HASH(
+   MENU_ENUM_LABEL_VALUE_AI_SERVICE_TEXT_NARRATOR_MODE,
+   "Text + Narrator"
+   )
+MSG_HASH(
+   MENU_ENUM_LABEL_VALUE_AI_SERVICE_IMAGE_NARRATOR_MODE,
+   "Image + Narrator"
+   )
+MSG_HASH(
+   MENU_ENUM_LABEL_VALUE_AI_SERVICE_TEXT_POSITION_BOTTOM,
+   "Bottom"
+   )
+MSG_HASH(
+   MENU_ENUM_LABEL_VALUE_AI_SERVICE_TEXT_POSITION_TOP,
+   "Top"
+   )
 MSG_HASH(
    MENU_ENUM_LABEL_VALUE_PLAYLIST_ENTRY_REMOVE_ENABLE_HIST_FAV,
    "History & Favorites"
@@ -13239,6 +13283,22 @@ MSG_HASH( /* FIXME Should be MSG_ */
    MENU_ENUM_LABEL_VALUE_SIDELOAD_CORE_ERROR,
    "Core installation failed"
    )
+MSG_HASH(
+   MSG_AI_VIDEO_DRIVER_NOT_SUPPORTED,
+   "Video driver not supported for AI Service."
+   )
+MSG_HASH(
+   MSG_AI_AUTO_MODE_ENABLED,
+   "Automatic translation enabled."
+   )
+MSG_HASH(
+   MSG_AI_AUTO_MODE_DISABLED,
+   "Automatic translation disabled."
+   )
+MSG_HASH(
+   MSG_AI_NOTHING_TO_TRANSLATE,
+   "Nothing to translate."
+   )
 MSG_HASH(
    MSG_CHEAT_DELETE_ALL_INSTRUCTIONS,
    "Press right five times to delete all cheats."

menu/cbs/menu_cbs_sublabel.c

@@ -269,6 +269,9 @@ DEFAULT_SUBLABEL_MACRO(action_bind_sublabel_ai_service_target_lang,  MENU_ENUM_S
 DEFAULT_SUBLABEL_MACRO(action_bind_sublabel_ai_service_source_lang,  MENU_ENUM_SUBLABEL_AI_SERVICE_SOURCE_LANG)
 DEFAULT_SUBLABEL_MACRO(action_bind_sublabel_ai_service_url,  MENU_ENUM_SUBLABEL_AI_SERVICE_URL)
 DEFAULT_SUBLABEL_MACRO(action_bind_sublabel_ai_service_enable,  MENU_ENUM_SUBLABEL_AI_SERVICE_ENABLE)
+DEFAULT_SUBLABEL_MACRO(action_bind_sublabel_ai_service_poll_delay,  MENU_ENUM_SUBLABEL_AI_SERVICE_POLL_DELAY)
+DEFAULT_SUBLABEL_MACRO(action_bind_sublabel_ai_service_text_position,  MENU_ENUM_SUBLABEL_AI_SERVICE_TEXT_POSITION)
+DEFAULT_SUBLABEL_MACRO(action_bind_sublabel_ai_service_text_padding,  MENU_ENUM_SUBLABEL_AI_SERVICE_TEXT_PADDING)
 DEFAULT_SUBLABEL_MACRO(action_bind_sublabel_power_management_settings_list,  MENU_ENUM_SUBLABEL_POWER_MANAGEMENT_SETTINGS)
 DEFAULT_SUBLABEL_MACRO(action_bind_sublabel_privacy_settings_list,         MENU_ENUM_SUBLABEL_PRIVACY_SETTINGS)
 DEFAULT_SUBLABEL_MACRO(action_bind_sublabel_midi_settings_list,            MENU_ENUM_SUBLABEL_MIDI_SETTINGS)
@@ -5001,6 +5004,15 @@ int menu_cbs_init_bind_sublabel(menu_file_list_cbs_t *cbs,
          case MENU_ENUM_LABEL_AI_SERVICE_ENABLE:
             BIND_ACTION_SUBLABEL(cbs, action_bind_sublabel_ai_service_enable);
             break;
+         case MENU_ENUM_LABEL_AI_SERVICE_POLL_DELAY:
+            BIND_ACTION_SUBLABEL(cbs, action_bind_sublabel_ai_service_poll_delay);
+            break;
+         case MENU_ENUM_LABEL_AI_SERVICE_TEXT_POSITION:
+            BIND_ACTION_SUBLABEL(cbs, action_bind_sublabel_ai_service_text_position);
+            break;
+         case MENU_ENUM_LABEL_AI_SERVICE_TEXT_PADDING:
+            BIND_ACTION_SUBLABEL(cbs, action_bind_sublabel_ai_service_text_padding);
+            break;
          case MENU_ENUM_LABEL_AI_SERVICE_SETTINGS:
             BIND_ACTION_SUBLABEL(cbs, action_bind_sublabel_ai_service_settings_list);
             break;

menu/menu_displaylist.c

@@ -5938,12 +5938,14 @@ void menu_displaylist_info_init(menu_displaylist_info_t *info)
    info->setting                  = NULL;
 }
 
-typedef struct menu_displaylist_build_info {
+typedef struct menu_displaylist_build_info 
+{
    enum msg_hash_enums enum_idx;
    enum menu_displaylist_parse_type parse_type;
 } menu_displaylist_build_info_t;
 
-typedef struct menu_displaylist_build_info_selective {
+typedef struct menu_displaylist_build_info_selective 
+{
    enum msg_hash_enums enum_idx;
    enum menu_displaylist_parse_type parse_type;
    bool checked;
@@ -6683,7 +6685,8 @@ unsigned menu_displaylist_build_list(
             bool playlist_show_sublabels = settings->bools.playlist_show_sublabels;
             bool history_list_enable     = settings->bools.history_list_enable;
             bool truncate_playlist       = settings->bools.ozone_truncate_playlist_name;
-            menu_displaylist_build_info_selective_t build_list[] = {
+            menu_displaylist_build_info_selective_t build_list[] = 
+            {
                {MENU_ENUM_LABEL_HISTORY_LIST_ENABLE,                 PARSE_ONLY_BOOL, true},
                {MENU_ENUM_LABEL_CONTENT_HISTORY_SIZE,                PARSE_ONLY_UINT, false},
                {MENU_ENUM_LABEL_CONTENT_FAVORITES_SIZE,              PARSE_ONLY_INT,  true},
@@ -7740,8 +7743,11 @@ unsigned menu_displaylist_build_list(
                {MENU_ENUM_LABEL_AI_SERVICE_MODE,          PARSE_ONLY_UINT,   false},
                {MENU_ENUM_LABEL_AI_SERVICE_URL,           PARSE_ONLY_STRING, false},
                {MENU_ENUM_LABEL_AI_SERVICE_PAUSE,         PARSE_ONLY_BOOL,   false},
+               {MENU_ENUM_LABEL_AI_SERVICE_POLL_DELAY,    PARSE_ONLY_UINT,   false},
                {MENU_ENUM_LABEL_AI_SERVICE_SOURCE_LANG,   PARSE_ONLY_UINT,   false},
                {MENU_ENUM_LABEL_AI_SERVICE_TARGET_LANG,   PARSE_ONLY_UINT,   false},
+               {MENU_ENUM_LABEL_AI_SERVICE_TEXT_POSITION, PARSE_ONLY_UINT,   false},
+               {MENU_ENUM_LABEL_AI_SERVICE_TEXT_PADDING,  PARSE_ONLY_UINT,   false},
             };
 
             for (i = 0; i < ARRAY_SIZE(build_list); i++)
@@ -7751,8 +7757,11 @@ unsigned menu_displaylist_build_list(
                   case MENU_ENUM_LABEL_AI_SERVICE_MODE:
                   case MENU_ENUM_LABEL_AI_SERVICE_URL:
                   case MENU_ENUM_LABEL_AI_SERVICE_PAUSE:
+                  case MENU_ENUM_LABEL_AI_SERVICE_POLL_DELAY:
                   case MENU_ENUM_LABEL_AI_SERVICE_SOURCE_LANG:
                   case MENU_ENUM_LABEL_AI_SERVICE_TARGET_LANG:
+                  case MENU_ENUM_LABEL_AI_SERVICE_TEXT_POSITION:
+                  case MENU_ENUM_LABEL_AI_SERVICE_TEXT_PADDING:
                      if (ai_service_enable)
                         build_list[i].checked = true;
                      break;

menu/menu_setting.c

@@ -3014,6 +3014,42 @@ static void setting_get_string_representation_uint_ai_service_mode(
       case 2:
          enum_idx = MENU_ENUM_LABEL_VALUE_AI_SERVICE_NARRATOR_MODE;
          break;
+      case 3:
+         enum_idx = MENU_ENUM_LABEL_VALUE_AI_SERVICE_TEXT_MODE;
+         break;
+      case 4:
+         enum_idx = MENU_ENUM_LABEL_VALUE_AI_SERVICE_TEXT_NARRATOR_MODE;
+         break;
+      case 5:
+         enum_idx = MENU_ENUM_LABEL_VALUE_AI_SERVICE_IMAGE_NARRATOR_MODE;
+         break;
+      default:
+         break;
+   }
+
+   if (enum_idx != 0)
+      strlcpy(s, msg_hash_to_str(enum_idx), len);
+}
+
+static void setting_get_string_representation_uint_ai_service_text_position(
+      rarch_setting_t *setting,
+      char *s, size_t len)
+{
+   enum msg_hash_enums enum_idx = MSG_UNKNOWN;
+   if (!setting)
+      return;
+
+   switch (*setting->value.target.unsigned_integer)
+   {
+      case 0:
+         enum_idx = MENU_ENUM_LABEL_VALUE_NONE;
+         break;
+      case 1:
+         enum_idx = MENU_ENUM_LABEL_VALUE_AI_SERVICE_TEXT_POSITION_BOTTOM;
+         break;
+      case 2:
+         enum_idx = MENU_ENUM_LABEL_VALUE_AI_SERVICE_TEXT_POSITION_TOP;
+         break;
       default:
          break;
    }
@@ -19234,7 +19270,7 @@ static bool setting_append_list(
          (*list)[list_info->index - 1].get_string_representation =
             &setting_get_string_representation_uint_ai_service_mode;
          (*list)[list_info->index - 1].action_ok     = &setting_action_ok_uint;
-         menu_settings_list_current_add_range(list, list_info, 0, 2, 1, true, true);
+         menu_settings_list_current_add_range(list, list_info, 0, 5, 1, true, true);
 
          CONFIG_STRING(
                list, list_info,
@@ -19317,6 +19353,50 @@ static bool setting_append_list(
          (*list)[list_info->index - 1].action_ok     = &setting_action_ok_uint;
          menu_settings_list_current_add_range(list, list_info, TRANSLATION_LANG_DONT_CARE, (TRANSLATION_LANG_LAST-1), 1, true, true);
          
+         CONFIG_UINT(
+               list, list_info,
+               &settings->uints.ai_service_poll_delay,
+               MENU_ENUM_LABEL_AI_SERVICE_POLL_DELAY,
+               MENU_ENUM_LABEL_VALUE_AI_SERVICE_POLL_DELAY,
+               DEFAULT_AI_SERVICE_POLL_DELAY,
+               &group_info,
+               &subgroup_info,
+               parent_group,
+               general_write_handler,
+               general_read_handler);
+         (*list)[list_info->index - 1].action_ok     = &setting_action_ok_uint;
+         menu_settings_list_current_add_range(list, list_info, 0, MAXIMUM_AI_SERVICE_POLL_DELAY, 50, true, true);
+         
+         CONFIG_UINT(
+               list, list_info,
+               &settings->uints.ai_service_text_position,
+               MENU_ENUM_LABEL_AI_SERVICE_TEXT_POSITION,
+               MENU_ENUM_LABEL_VALUE_AI_SERVICE_TEXT_POSITION,
+               DEFAULT_AI_SERVICE_TEXT_POSITION,
+               &group_info,
+               &subgroup_info,
+               parent_group,
+               general_write_handler,
+               general_read_handler);
+         (*list)[list_info->index - 1].get_string_representation =
+            &setting_get_string_representation_uint_ai_service_text_position;
+         (*list)[list_info->index - 1].action_ok     = &setting_action_ok_uint;
+         menu_settings_list_current_add_range(list, list_info, 0, 2, 1, true, true);
+         
+         CONFIG_UINT(
+               list, list_info,
+               &settings->uints.ai_service_text_padding,
+               MENU_ENUM_LABEL_AI_SERVICE_TEXT_PADDING,
+               MENU_ENUM_LABEL_VALUE_AI_SERVICE_TEXT_PADDING,
+               DEFAULT_AI_SERVICE_TEXT_PADDING,
+               &group_info,
+               &subgroup_info,
+               parent_group,
+               general_write_handler,
+               general_read_handler);
+         (*list)[list_info->index - 1].action_ok     = &setting_action_ok_uint;
+         menu_settings_list_current_add_range(list, list_info, 0, 20, 1, true, true);
+
          END_SUB_GROUP(list, list_info, parent_group);
          END_GROUP(list, list_info, parent_group);
 #endif

msg_hash.h

@@ -569,6 +569,10 @@ enum msg_hash_enums
    MSG_FAILED_TO_ENTER_GAMEMODE_LINUX,
    MSG_VRR_RUNLOOP_ENABLED,
    MSG_VRR_RUNLOOP_DISABLED,
+   MSG_AI_VIDEO_DRIVER_NOT_SUPPORTED,
+   MSG_AI_AUTO_MODE_ENABLED,
+   MSG_AI_AUTO_MODE_DISABLED,
+   MSG_AI_NOTHING_TO_TRANSLATE,
    MSG_VIDEO_REFRESH_RATE_CHANGED,
 
    MSG_IOS_TOUCH_MOUSE_ENABLED,
@@ -2785,6 +2789,9 @@ enum msg_hash_enums
    MENU_LABEL(AI_SERVICE_URL),
    MENU_LABEL(AI_SERVICE_ENABLE),
    MENU_LABEL(AI_SERVICE_PAUSE),
+   MENU_LABEL(AI_SERVICE_POLL_DELAY),
+   MENU_LABEL(AI_SERVICE_TEXT_POSITION),
+   MENU_LABEL(AI_SERVICE_TEXT_PADDING),
 
    MENU_LABEL(ON),
    MENU_LABEL(OFF),
@@ -3477,6 +3484,11 @@ enum msg_hash_enums
    MENU_ENUM_LABEL_VALUE_AI_SERVICE_IMAGE_MODE,
    MENU_ENUM_LABEL_VALUE_AI_SERVICE_SPEECH_MODE,
    MENU_ENUM_LABEL_VALUE_AI_SERVICE_NARRATOR_MODE,
+   MENU_ENUM_LABEL_VALUE_AI_SERVICE_TEXT_MODE,
+   MENU_ENUM_LABEL_VALUE_AI_SERVICE_TEXT_NARRATOR_MODE,
+   MENU_ENUM_LABEL_VALUE_AI_SERVICE_IMAGE_NARRATOR_MODE,
+   MENU_ENUM_LABEL_VALUE_AI_SERVICE_TEXT_POSITION_TOP,
+   MENU_ENUM_LABEL_VALUE_AI_SERVICE_TEXT_POSITION_BOTTOM,
 
    MENU_ENUM_LABEL_VALUE_NONE,
    MENU_ENUM_LABEL_VALUE_NO_INFORMATION_AVAILABLE,

retroarch.c

@@ -2236,6 +2236,9 @@ bool command_event(enum event_command cmd, void *data)
 #if defined(HAVE_ACCESSIBILITY) || defined(HAVE_TRANSLATE)
    access_state_t *access_st       = access_state_get_ptr();
 #endif
+#if defined(HAVE_TRANSLATE) && defined(HAVE_GFX_WIDGETS)
+   dispgfx_widget_t *p_dispwidget  = dispwidget_get_ptr();
+#endif
 #ifdef HAVE_MENU
    struct menu_state *menu_st      = menu_state_get_ptr();
 #endif
@@ -2252,12 +2255,12 @@ bool command_event(enum event_command cmd, void *data)
 #ifdef HAVE_OVERLAY
          input_overlay_unload();
 #endif
-#if defined(HAVE_TRANSLATE) && defined(HAVE_GFX_WIDGETS)
+#ifdef HAVE_TRANSLATE
-         /* Because the overlay is a display widget,
+         translation_release(true);
-          * it's going to be written
+#ifdef HAVE_GFX_WIDGETS
-          * over the menu, so we unset it here. */
+         if (p_dispwidget->ai_service_overlay_state != 0)
-         if (dispwidget_get_ptr()->ai_service_overlay_state != 0)
             gfx_widgets_ai_service_overlay_unload();
+#endif
 #endif
          break;
       case CMD_EVENT_OVERLAY_INIT:
@@ -2331,6 +2334,11 @@ bool command_event(enum event_command cmd, void *data)
                            accessibility_narrator_speech_speed,
                            (char*)msg_hash_to_str(MSG_UNPAUSED), 10);
 #endif
+#ifdef HAVE_GFX_WIDGETS
+                  if (p_dispwidget->ai_service_overlay_state != 0)
+                     gfx_widgets_ai_service_overlay_unload();
+#endif
+                  translation_release(true);
                   command_event(CMD_EVENT_UNPAUSE, NULL);
                }
                else /* Pause on call */
@@ -2349,18 +2357,26 @@ bool command_event(enum event_command cmd, void *data)
                 * Also, this mode is required for "auto" translation
                 * packages, since you don't want to pause for that.
                 */
-               if (access_st->ai_service_auto == 2)
+               if (access_st->ai_service_auto != 0)
                {
                   /* Auto mode was turned on, but we pressed the
                    * toggle button, so turn it off now. */
-                  access_st->ai_service_auto = 0;
+                  translation_release(true);
-#ifdef HAVE_MENU_WIDGETS
+#ifdef HAVE_GFX_WIDGETS
+                  if (p_dispwidget->ai_service_overlay_state != 0)
                      gfx_widgets_ai_service_overlay_unload();
 #endif
                }
                else 
+               {
+#ifdef HAVE_GFX_WIDGETS
+                  if (p_dispwidget->ai_service_overlay_state != 0)
+                     gfx_widgets_ai_service_overlay_unload();
+                  else
+#endif
                      command_event(CMD_EVENT_AI_SERVICE_CALL, NULL);
                }
+            }
 #endif
             break;
          }
@@ -4473,10 +4489,6 @@ bool command_event(enum event_command cmd, void *data)
                if (data)
                   paused = *((bool*)data);
 
-               if (     (access_st->ai_service_auto == 0)
-                     && !settings->bools.ai_service_pause)
-                  access_st->ai_service_auto = 1;
-
                run_translation_service(settings, paused);
             }
 #endif
@@ -7165,6 +7177,9 @@ bool retroarch_main_quit(void)
    video_driver_state_t*video_st = video_state_get_ptr();
    settings_t *settings          = config_get_ptr();
    bool config_save_on_exit      = settings->bools.config_save_on_exit;
+#ifdef HAVE_ACCESSIBILITY
+   access_state_t *access_st     = access_state_get_ptr();
+#endif
    struct retro_system_av_info *av_info = &video_st->av_info;
 
    /* Restore video driver before saving */
@@ -7263,6 +7278,17 @@ bool retroarch_main_quit(void)
    retroarch_menu_running_finished(true);
 #endif
 
+#ifdef HAVE_ACCESSIBILITY
+   translation_release(false);
+#ifdef HAVE_THREADS
+   if (access_st->image_lock)
+   {
+      slock_free(access_st->image_lock);
+      access_st->image_lock = NULL;
+   }
+#endif
+#endif
+
    return true;
 }