AI Service feature.

command.c

@@ -242,7 +242,7 @@ static const struct cmd_map map[] = {
    { "MENU_RIGHT",             RETRO_DEVICE_ID_JOYPAD_RIGHT },
    { "MENU_A",                 RETRO_DEVICE_ID_JOYPAD_A },
    { "MENU_B",                 RETRO_DEVICE_ID_JOYPAD_B },
-   { "MENU_B",                 RETRO_DEVICE_ID_JOYPAD_B },
+   { "AI_SERVICE",             RARCH_AI_SERVICE },
 };
 #endif
 
@@ -1767,8 +1767,34 @@ bool command_event(enum event_command cmd, void *data)
          bsv_movie_check();
          break;
       case CMD_EVENT_AI_SERVICE_TOGGLE:
-         /* TODO/FIXME - implement */
+      {
+         settings_t *settings      = config_get_ptr();
+#ifdef HAVE_TRANSLATE
+         if (settings->uints.ai_service_mode == 0)
+         {
+            /* Default mode - pause on call, unpause on second press. */
+            if (!rarch_ctl(RARCH_CTL_IS_PAUSED, NULL))
+            {
+               command_event(CMD_EVENT_PAUSE, NULL);
+               command_event(CMD_EVENT_AI_SERVICE_CALL, NULL);
+            }
+            else
+            {
+               command_event(CMD_EVENT_UNPAUSE, NULL);
+            }
+         }
+         else if (settings->uints.ai_service_mode == 1)
+         {
+            /* Text-to-Speech mode - don't pause */
+            command_event(CMD_EVENT_AI_SERVICE_CALL, NULL);
+         }
+         else
+         {
+            RARCH_LOG("Invalid AI Service Mode.\n");
+         }
+#endif
          break;
+      }
       case CMD_EVENT_STREAMING_TOGGLE:
          if (streaming_is_enabled())
             command_event(CMD_EVENT_RECORD_DEINIT, NULL);
@@ -2524,7 +2550,6 @@ TODO: Add a setting for these tweaks */
             bool is_idle              = false;
             bool is_slowmotion        = false;
             bool is_perfcnt_enable    = false;
-            settings_t *settings      = config_get_ptr();
 
 #ifdef HAVE_DISCORD
             discord_userdata_t userdata;
@@ -2548,22 +2573,6 @@ TODO: Add a setting for these tweaks */
                if (!is_idle)
                   video_driver_cached_frame();
 
-               /* If OCR enabled, translate the screen while paused */
-               if (settings->bools.translation_service_enable)
-               {
-#ifdef HAVE_TRANSLATE
-                  if (!g_translation_service_status)
-                  {
-                     RARCH_LOG("OCR START\n");
-                     run_translation_service();
-                     g_translation_service_status = true;
-                  }
-#else
-                  RARCH_LOG("OCR Translation not enabled in build.  Include HAVE_TRANSLATE define.\n");
-#endif
-               }
-
-
 #ifdef HAVE_DISCORD
                userdata.status = DISCORD_PRESENCE_GAME_PAUSED;
                command_event(CMD_EVENT_DISCORD_UPDATE, &userdata);
@@ -2573,9 +2582,6 @@ TODO: Add a setting for these tweaks */
             {
 #if defined(HAVE_MENU) && defined(HAVE_MENU_WIDGETS)
                menu_widgets_set_paused(is_paused);
-#endif
-#ifdef HAVE_TRANSLATE
-               g_translation_service_status = false;
 #endif
                RARCH_LOG("%s\n", msg_hash_to_str(MSG_UNPAUSED));
                command_event(CMD_EVENT_AUDIO_START, NULL);
@@ -3094,6 +3100,14 @@ TODO: Add a setting for these tweaks */
          }
 #endif
          break;
+
+      case CMD_EVENT_AI_SERVICE_CALL:
+         {
+#ifdef HAVE_TRANSLATE
+            RARCH_LOG("AI Service Called...\n");
+            run_translation_service();
+#endif
+         }
       case CMD_EVENT_NONE:
          return false;
    }

command.h

@@ -255,7 +255,8 @@ enum event_command
    CMD_EVENT_SHADER_PREV,
    CMD_EVENT_CHEAT_INDEX_PLUS,
    CMD_EVENT_CHEAT_INDEX_MINUS,
-   CMD_EVENT_CHEAT_TOGGLE
+   CMD_EVENT_CHEAT_TOGGLE,
+   CMD_EVENT_AI_SERVICE_CALL
 };
 
 bool command_set_shader(const char *arg);

config.def.h

@@ -457,8 +457,6 @@ static bool menu_swap_ok_cancel_buttons = false;
 
 static bool quit_press_twice = false;
 
-static bool default_translation_service_enable = false;
-
 static bool default_log_to_file = false;
 static bool log_to_file_timestamp = false;
 
@@ -982,6 +980,6 @@ static char buildbot_assets_server_url[] = "http://buildbot.libretro.com/assets/
 
 static char default_discord_app_id[] = "475456035851599874";
 
-static char default_translation_service_url[] = "http://localhost:4404/";
+static char default_ai_service_url[] = "http://localhost:4404/";
 
 #endif

config.def.keybinds.h

@@ -104,6 +104,7 @@ static const struct retro_keybind retro_keybinds_1[] = {
    { true, RARCH_MENU_TOGGLE,              MENU_ENUM_LABEL_VALUE_INPUT_META_MENU_TOGGLE,          RETROK_SPACE,   NO_BTN, NO_BTN, 0, AXIS_NONE },
    { true, RARCH_RECORDING_TOGGLE,         MENU_ENUM_LABEL_VALUE_INPUT_META_RECORDING_TOGGLE,     RETROK_UNKNOWN,      NO_BTN, NO_BTN, 0, AXIS_NONE },
    { true, RARCH_STREAMING_TOGGLE,         MENU_ENUM_LABEL_VALUE_INPUT_META_STREAMING_TOGGLE,     RETROK_UNKNOWN,      NO_BTN, NO_BTN, 0, AXIS_NONE },
+   { true, RARCH_AI_SERVICE,               MENU_ENUM_LABEL_VALUE_INPUT_META_AI_SERVICE,     RETROK_UNKNOWN,            NO_BTN, NO_BTN, 0, AXIS_NONE },
 #else
    { true, RETRO_DEVICE_ID_JOYPAD_B,      MENU_ENUM_LABEL_VALUE_INPUT_JOYPAD_B,              RETROK_z,       NO_BTN, NO_BTN, 0, AXIS_NONE },
    { true, RETRO_DEVICE_ID_JOYPAD_Y,      MENU_ENUM_LABEL_VALUE_INPUT_JOYPAD_Y,              RETROK_a,       NO_BTN, NO_BTN, 0, AXIS_NONE },
@@ -185,6 +186,8 @@ static const struct retro_keybind retro_keybinds_1[] = {
    { true, RARCH_MENU_TOGGLE,              MENU_ENUM_LABEL_VALUE_INPUT_META_MENU_TOGGLE,          RETROK_F1,      NO_BTN, NO_BTN, 0, AXIS_NONE },
    { true, RARCH_RECORDING_TOGGLE,         MENU_ENUM_LABEL_VALUE_INPUT_META_RECORDING_TOGGLE,     RETROK_UNKNOWN,      NO_BTN, NO_BTN, 0, AXIS_NONE },
    { true, RARCH_STREAMING_TOGGLE,         MENU_ENUM_LABEL_VALUE_INPUT_META_STREAMING_TOGGLE,     RETROK_UNKNOWN,      NO_BTN, NO_BTN, 0, AXIS_NONE },
+   { true, RARCH_AI_SERVICE,               MENU_ENUM_LABEL_VALUE_INPUT_META_AI_SERVICE,     RETROK_UNKNOWN,            NO_BTN, NO_BTN, 0, AXIS_NONE },
+
 #endif
 };
 

configuration.c

@@ -1167,8 +1167,7 @@ static struct config_array_setting *populate_settings_array(settings_t *settings
    SETTING_ARRAY("youtube_stream_key",       settings->arrays.youtube_stream_key, true, NULL, true);
    SETTING_ARRAY("twitch_stream_key",       settings->arrays.twitch_stream_key, true, NULL, true);
    SETTING_ARRAY("discord_app_id",           settings->arrays.discord_app_id, true, default_discord_app_id, true);
-   SETTING_ARRAY("translation_service_url",  settings->arrays.translation_service_url, true, default_translation_service_url, true);
+   SETTING_ARRAY("ai_service_url",           settings->arrays.ai_service_url, true, default_ai_service_url, true);
-   SETTING_ARRAY("ai_service_url", settings->arrays.ai_service_url, true, "", true);
 
    *size = count;
 
@@ -1616,8 +1615,6 @@ static struct config_bool_setting *populate_settings_bool(settings_t *settings,
 #ifdef HAVE_OZONE
    SETTING_BOOL("ozone_collapse_sidebar",       &settings->bools.ozone_collapse_sidebar, true, DEFAULT_OZONE_COLLAPSE_SIDEBAR, false);
 #endif
-   SETTING_BOOL("translation_service_enable", &settings->bools.translation_service_enable, true, default_translation_service_enable, false);
-
    SETTING_BOOL("log_to_file", &settings->bools.log_to_file, true, default_log_to_file, false);
    SETTING_OVERRIDE(RARCH_OVERRIDE_SETTING_LOG_TO_FILE);
    SETTING_BOOL("log_to_file_timestamp", &settings->bools.log_to_file_timestamp, true, log_to_file_timestamp, false);
@@ -2022,8 +2019,8 @@ void config_set_defaults(void)
    strlcpy(settings->arrays.discord_app_id,
       default_discord_app_id,  sizeof(settings->arrays.discord_app_id));
 
-   strlcpy(settings->arrays.translation_service_url,
+   strlcpy(settings->arrays.ai_service_url,
-      default_translation_service_url,  sizeof(settings->arrays.translation_service_url));
+      default_ai_service_url,  sizeof(settings->arrays.ai_service_url));
 
 
 #ifdef HAVE_MATERIALUI

configuration.h

@@ -334,9 +334,6 @@ typedef struct settings
       bool enable_device_vibration;
       bool ozone_collapse_sidebar;
 
-
-      bool translation_service_enable;
-
       bool log_to_file;
       bool log_to_file_timestamp;
 

retroarch.c

@@ -2215,7 +2215,7 @@ const struct input_bind_map input_config_bind_map[RARCH_BIND_LIST_END_NULL] = {
 #endif
       DECLARE_META_BIND(2, recording_toggle,      RARCH_RECORDING_TOGGLE,      MENU_ENUM_LABEL_VALUE_INPUT_META_RECORDING_TOGGLE),
       DECLARE_META_BIND(2, streaming_toggle,      RARCH_STREAMING_TOGGLE,      MENU_ENUM_LABEL_VALUE_INPUT_META_STREAMING_TOGGLE),
-      DECLARE_META_BIND(2, streaming_toggle,      RARCH_AI_SERVICE,      MENU_ENUM_LABEL_VALUE_INPUT_META_AI_SERVICE),
+      DECLARE_META_BIND(2, ai_service,            RARCH_AI_SERVICE,            MENU_ENUM_LABEL_VALUE_INPUT_META_AI_SERVICE),
 };
 
 typedef struct turbo_buttons turbo_buttons_t;
@@ -16900,7 +16900,6 @@ void rarch_force_video_driver_fallback(const char *driver)
 
       free(title);
    }
-
    exit(1);
 }
 

translation/translation_service.c

@@ -7,12 +7,28 @@
 #include "gfx/video_frame.h"
 #include "gfx/scaler/scaler.h"
 #include "tasks/tasks_internal.h"
+#include <audio/audio_mixer.h>
+#include "tasks/task_audio_mixer.h"
 
 #include "configuration.h"
 #include "retroarch.h"
 #include "verbosity.h"
 
+#ifdef HAVE_TRANSLATE
+#include "translation/translation_service.h"
+#endif
+
+
+typedef struct nbio_buf
+{
+   void *buf;
+   unsigned bufsize;
+   char *path;
+} nbio_buf_t;
+
+/*
 bool g_translation_service_status = false;
+*/
 
 static void form_bmp_header(uint8_t *header, unsigned width, unsigned height,
                             bool is32bpp)
@@ -103,8 +119,8 @@ bool run_translation_service(void)
       image to the screen.
 
       Supported client/services (thus far)
-      -Ztranslate client/service ( www.ztranslate.net/docs/service )
       -VGTranslate client ( www.gitlab.com/spherebeaker/vg_translate )
+      -Ztranslate client/service ( www.ztranslate.net/docs/service )
       
       To use a client, download the relevant code/release, configure
       them, and run them on your local machine, or network.  Set the 
@@ -116,7 +132,6 @@ bool run_translation_service(void)
       downside here is that your retroarch device will have to have
       an internet connection, and you may have to sign up for it.
      
-      
       To make your own server, it must listen for a POST request, which
       will consist of a json body, with the "image" field as a base64
       encoded string of a 24bit-BMP that the will be translated.  The server
@@ -307,17 +322,20 @@ void handle_translation_cb(retro_task_t *task, void *task_data, void *user_data,
    char* ret;
    char* string                      = NULL;
 
-   int new_size                      = 0;
+   int new_image_size                = 0;
+   int new_sound_size                = 0;
    unsigned width, height;
    unsigned image_width, image_height;
    size_t pitch;
    const void* dummy_data;
-   void* raw_image_data;  
+   void* raw_image_data              = NULL;  
+   void* raw_sound_data              = NULL;  
+   settings_t *settings              = config_get_ptr();
 
    runloop_get_status(&is_paused, &is_idle, &is_slowmotion,
        &is_perfcnt_enable);
 
-   if (!is_paused)
+   if (!is_paused && settings->uints.ai_service_mode != 1)
       goto finish;
 
    if (!data || error)
@@ -329,41 +347,73 @@ void handle_translation_cb(retro_task_t *task, void *task_data, void *user_data,
 
    data->data[data->len] = '\0'; 
 
-   /* Parse JSON body for the image data */
+   /* Parse JSON body for the image and sound data */
-   body_copy = strdup(strchr(data->data, ch));
-   ret = body_copy;
-   if (!ret)
-      goto finish;
 
-   while (strncmp(ret, "\"image\":", strlen("\"image\":"))!=0)
+   body_copy = strdup(data->data);
+   int i = 0;
+   int start = -1;
+   char* found_string = NULL;
+   int curr_state = 0;
+   char curr;
+
+   while (true)
    {
-      ret = strchr(ret, ch);
+      curr = (char) *(body_copy+i);
-      if (ret == NULL)
+      if (curr == '\0')
           break;
-      else
+      if (curr == '\"')
-         ret = ret+sizeof(char);
-   }
-   
-   if (ret != NULL)
       {
-      ret = ret + sizeof(char);
+         if (start == -1)
-      ret = ret+strlen("\"image\":")*sizeof(char);
+            start = i;
-
+         else
-      ret = strchr(ret, ch)+sizeof(char);
+         {
-      string = strtok(ret, s);
+            found_string = malloc(i-start);
+            strncpy(found_string, body_copy+start+1, i-start-1);
+            *(found_string+i-start-1) = '\0';
+            if (curr_state == 1)/*image*/
+            {
+              raw_bmp_data = (void*) unbase64(found_string, 
+                                              strlen(found_string),
+                                              &new_image_size);
+              curr_state = 0;
             }
+            else if (curr_state == 2)
+            {
+              raw_sound_data = (void*) unbase64(found_string, 
+                                                strlen(found_string),
+                                                &new_sound_size);           
+              curr_state = 0;
+            }
+            else if (strcmp(found_string, "image")==0)
+            {
+              curr_state = 1;
+              free(found_string);
+            }
+            else if (strcmp(found_string, "sound")==0)
+            {
+              curr_state = 2;
+              free(found_string);
+            }
+            else
+            {
+              curr_state = 0;
+            }
+            start = -1;
+         }
+      }
+      i++;
+   }
+   if (found_string)
+       free(found_string);
 
-   if (ret == NULL || string == NULL)
+   if (raw_bmp_data == NULL && raw_sound_data == NULL)
    {
       error = "Invalid JSON body.";
       goto finish;
    }
 
-   /* decode the image data from base64 */
+   if (raw_bmp_data != NULL)
-   raw_bmp_data = (void*) unbase64(string, strlen(string), &new_size);
+   { 
-   if (!raw_bmp_data)
-      goto finish;
-
       /* Get the video frame dimensions reference */
       video_driver_cached_frame_get(&dummy_data, &width, &height, &pitch);
   
@@ -392,7 +442,7 @@ void handle_translation_cb(retro_task_t *task, void *task_data, void *user_data,
          */
 
          /* TODO: write to the viewport in this case */
-      RARCH_LOG("WRITING TO VIEWPORT...\n");
+         RARCH_LOG("Hardware frame buffer... writing to viewport not yet supported.\n");
          goto finish;
       }
 
@@ -428,12 +478,48 @@ void handle_translation_cb(retro_task_t *task, void *task_data, void *user_data,
       scaler_ctx_gen_filter(scaler);
       scaler->in_stride = -1*width*3;
 
-   scaler_ctx_scale_direct(scaler, raw_output_data, (uint8_t*)raw_image_data+(image_height-1)*width*3);
+      scaler_ctx_scale_direct(scaler, raw_output_data, 
-   /*
+                              (uint8_t*)raw_image_data+(image_height-1)*width*3);
-   video_driver_frame(raw_output_data+(height-1)*pitch, image_width, image_height, -pitch);
-   */
       video_driver_frame(raw_output_data, image_width, image_height, pitch);
+   }
+   if (raw_sound_data != NULL)
+   {
+     
+      retro_task_t *t = task_init();
+      nbio_buf_t *task_data = (nbio_buf_t*)calloc(1, sizeof(nbio_buf_t));
+      task_data->buf = raw_sound_data;
+      task_data->bufsize = new_sound_size;
+      task_data->path=NULL;
+      
+   audio_mixer_stream_params_t params;
+   nbio_buf_t *img = (nbio_buf_t*)task_data;
+  
+   if (!img)
+      return;
+   
+   params.volume               = 1.0f;
+   params.slot_selection_type  = AUDIO_MIXER_SLOT_SELECTION_AUTOMATIC;//user->slot_selection_type;
+   params.slot_selection_idx   = 10;
+   params.stream_type          = AUDIO_STREAM_TYPE_SYSTEM;//user->stream_type;
+   params.type                 = AUDIO_MIXER_TYPE_WAV;
+   params.state                = AUDIO_STREAM_STATE_PLAYING;
+   params.buf                  = img->buf;
+   params.bufsize              = img->bufsize;
+   params.cb                   = NULL;
+   params.basename             = NULL;
+
+   audio_driver_mixer_add_stream(&params);
+
+   if (img->path)
+      free(img->path);
+   if (params.basename != NULL)
+      free(params.basename);
+   free(img);
+
+      RARCH_LOG("Loaded Sound Size: %i\n", new_sound_size);
+   }
    RARCH_LOG("Translation done.\n"); 
+
 finish:
    if (error)
       RARCH_ERR("%s: %s\n", msg_hash_to_str(MSG_DOWNLOAD_FAILED), error);
@@ -465,7 +551,7 @@ void call_translation_server(const char* body)
 {
    settings_t *settings                  = config_get_ptr();
 
-   RARCH_LOG("Server url:  %s\n", settings->arrays.translation_service_url);
+   RARCH_LOG("Server url:  %s\n", settings->arrays.ai_service_url);
-   task_push_http_post_transfer(settings->arrays.translation_service_url, 
+   task_push_http_post_transfer(settings->arrays.ai_service_url, 
                                 body, true, NULL, handle_translation_cb, NULL);
 }

translation/translation_service.h

@@ -5,7 +5,9 @@
 
 void call_translation_server(const char* body);
 
+/*
 bool g_translation_service_status;
+*/
 
 bool run_translation_service(void);