Merge pull request #8905 from JosJuice/jni-encoding

Android: Use correct encoding when converting strings

Source/Android/jni/AndroidCommon/AndroidCommon.cpp

@@ -5,25 +5,28 @@
 #include "jni/AndroidCommon/AndroidCommon.h"
 
 #include <string>
+#include <string_view>
 #include <vector>
 
 #include <jni.h>
 
+#include "Common/StringUtil.h"
+
 std::string GetJString(JNIEnv* env, jstring jstr)
 {
-  std::string result = "";
-  if (!jstr)
-    return result;
-
-  const char* s = env->GetStringUTFChars(jstr, nullptr);
-  result = s;
-  env->ReleaseStringUTFChars(jstr, s);
-  return result;
+  const jchar* jchars = env->GetStringChars(jstr, nullptr);
+  const jsize length = env->GetStringLength(jstr);
+  const std::u16string_view string_view(reinterpret_cast<const char16_t*>(jchars), length);
+  const std::string converted_string = UTF16ToUTF8(string_view);
+  env->ReleaseStringChars(jstr, jchars);
+  return converted_string;
 }
 
 jstring ToJString(JNIEnv* env, const std::string& str)
 {
-  return env->NewStringUTF(str.c_str());
+  const std::u16string converted_string = UTF8ToUTF16(str);
+  return env->NewString(reinterpret_cast<const jchar*>(converted_string.data()),
+                        converted_string.size());
 }
 
 std::vector<std::string> JStringArrayToVector(JNIEnv* env, jobjectArray array)

Source/Core/Common/FileUtil.cpp

@@ -114,7 +114,7 @@ bool Exists(const std::string& path)
 bool IsDirectory(const std::string& path)
 {
 #ifdef _WIN32
-  return PathIsDirectory(UTF8ToUTF16(path).c_str());
+  return PathIsDirectory(UTF8ToWString(path).c_str());
 #else
   return FileInfo(path).IsDirectory();
 #endif

Source/Core/Common/Logging/ConsoleListenerWin.cpp

@@ -17,5 +17,5 @@ ConsoleListener::~ConsoleListener()
 
 void ConsoleListener::Log([[maybe_unused]] Common::Log::LOG_LEVELS level, const char* text)
 {
-  ::OutputDebugStringW(UTF8ToUTF16(text).c_str());
+  ::OutputDebugStringW(UTF8ToWString(text).c_str());
 }

Source/Core/Common/StringUtil.cpp

@@ -5,6 +5,7 @@
 #include "Common/StringUtil.h"
 
 #include <algorithm>
+#include <codecvt>
 #include <cstdarg>
 #include <cstddef>
 #include <cstdio>
@@ -17,6 +18,7 @@
 #include <locale>
 #include <sstream>
 #include <string>
+#include <type_traits>
 #include <vector>
 
 #include <fmt/format.h>
@@ -32,7 +34,6 @@
 constexpr u32 CODEPAGE_SHIFT_JIS = 932;
 constexpr u32 CODEPAGE_WINDOWS_1252 = 1252;
 #else
-#include <codecvt>
 #include <errno.h>
 #include <iconv.h>
 #include <locale.h>
@@ -463,29 +464,29 @@ std::string UTF16ToCP(u32 code_page, std::wstring_view input)
   return output;
 }
 
-std::wstring UTF8ToUTF16(std::string_view input)
+std::wstring UTF8ToWString(std::string_view input)
 {
   return CPToUTF16(CP_UTF8, input);
 }
 
-std::string UTF16ToUTF8(std::wstring_view input)
+std::string WStringToUTF8(std::wstring_view input)
 {
   return UTF16ToCP(CP_UTF8, input);
 }
 
 std::string SHIFTJISToUTF8(std::string_view input)
 {
-  return UTF16ToUTF8(CPToUTF16(CODEPAGE_SHIFT_JIS, input));
+  return WStringToUTF8(CPToUTF16(CODEPAGE_SHIFT_JIS, input));
 }
 
 std::string UTF8ToSHIFTJIS(std::string_view input)
 {
-  return UTF16ToCP(CODEPAGE_SHIFT_JIS, UTF8ToUTF16(input));
+  return UTF16ToCP(CODEPAGE_SHIFT_JIS, UTF8ToWString(input));
 }
 
 std::string CP1252ToUTF8(std::string_view input)
 {
-  return UTF16ToUTF8(CPToUTF16(CODEPAGE_WINDOWS_1252, input));
+  return WStringToUTF8(CPToUTF16(CODEPAGE_WINDOWS_1252, input));
 }
 
 std::string UTF16BEToUTF8(const char16_t* str, size_t max_size)
@@ -493,7 +494,7 @@ std::string UTF16BEToUTF8(const char16_t* str, size_t max_size)
   const char16_t* str_end = std::find(str, str + max_size, '\0');
   std::wstring result(static_cast<size_t>(str_end - str), '\0');
   std::transform(str, str_end, result.begin(), static_cast<u16 (&)(u16)>(Common::swap16));
-  return UTF16ToUTF8(result);
+  return WStringToUTF8(result);
 }
 
 #else
@@ -578,9 +579,12 @@ std::string UTF8ToSHIFTJIS(std::string_view input)
   return CodeTo("SJIS", "UTF-8", input);
 }
 
-std::string UTF16ToUTF8(std::wstring_view input)
+std::string WStringToUTF8(std::wstring_view input)
 {
-  std::wstring_convert<std::codecvt_utf8<wchar_t>, wchar_t> converter;
+  using codecvt = std::conditional_t<sizeof(wchar_t) == 2, std::codecvt_utf8_utf16<wchar_t>,
+                                     std::codecvt_utf8<wchar_t>>;
+
+  std::wstring_convert<codecvt, wchar_t> converter;
   return converter.to_bytes(input.data(), input.data() + input.size());
 }
 
@@ -592,12 +596,24 @@ std::string UTF16BEToUTF8(const char16_t* str, size_t max_size)
 
 #endif
 
+std::string UTF16ToUTF8(std::u16string_view input)
+{
+  std::wstring_convert<std::codecvt_utf8_utf16<char16_t>, char16_t> converter;
+  return converter.to_bytes(input.data(), input.data() + input.size());
+}
+
+std::u16string UTF8ToUTF16(std::string_view input)
+{
+  std::wstring_convert<std::codecvt_utf8_utf16<char16_t>, char16_t> converter;
+  return converter.from_bytes(input.data(), input.data() + input.size());
+}
+
 #ifdef HAS_STD_FILESYSTEM
 // This is a replacement for path::u8path, which is deprecated starting with C++20.
 std::filesystem::path StringToPath(std::string_view path)
 {
 #ifdef _MSC_VER
-  return std::filesystem::path(UTF8ToUTF16(path));
+  return std::filesystem::path(UTF8ToWString(path));
 #else
   return std::filesystem::path(path);
 #endif
@@ -608,7 +624,7 @@ std::filesystem::path StringToPath(std::string_view path)
 std::string PathToString(const std::filesystem::path& path)
 {
 #ifdef _MSC_VER
-  return UTF16ToUTF8(path.native());
+  return WStringToUTF8(path.native());
 #else
   return path.native();
 #endif

Source/Core/Common/StringUtil.h

@@ -172,22 +172,24 @@ size_t StringUTF8CodePointCount(const std::string& str);
 std::string CP1252ToUTF8(std::string_view str);
 std::string SHIFTJISToUTF8(std::string_view str);
 std::string UTF8ToSHIFTJIS(std::string_view str);
-std::string UTF16ToUTF8(std::wstring_view str);
+std::string WStringToUTF8(std::wstring_view str);
 std::string UTF16BEToUTF8(const char16_t* str, size_t max_size);  // Stops at \0
+std::string UTF16ToUTF8(std::u16string_view str);
+std::u16string UTF8ToUTF16(std::string_view str);
 
 #ifdef _WIN32
 
-std::wstring UTF8ToUTF16(std::string_view str);
+std::wstring UTF8ToWString(std::string_view str);
 
 #ifdef _UNICODE
 inline std::string TStrToUTF8(std::wstring_view str)
 {
-  return UTF16ToUTF8(str);
+  return WStringToUTF8(str);
 }
 
 inline std::wstring UTF8ToTStr(std::string_view str)
 {
-  return UTF8ToUTF16(str);
+  return UTF8ToWString(str);
 }
 #else
 inline std::string TStrToUTF8(std::string_view str)
@@ -221,7 +223,7 @@ std::string ThousandSeparate(I value, int spaces = 0)
   stream << std::setw(spaces) << value;
 
 #ifdef _WIN32
-  return UTF16ToUTF8(stream.str());
+  return WStringToUTF8(stream.str());
 #else
   return stream.str();
 #endif

Source/Core/Common/Timer.cpp

@@ -217,7 +217,7 @@ std::string Timer::GetTimeFormatted()
 #ifdef _WIN32
   struct timeb tp;
   (void)::ftime(&tp);
-  return UTF16ToUTF8(tmp) + fmt::format(":{:03}", tp.millitm);
+  return WStringToUTF8(tmp) + fmt::format(":{:03}", tp.millitm);
 #elif defined __APPLE__
   struct timeval t;
   (void)gettimeofday(&t, nullptr);
@@ -255,7 +255,7 @@ std::string Timer::GetDateTimeFormatted(double time)
 #ifdef _WIN32
   wchar_t tmp[32] = {};
   wcsftime(tmp, sizeof(tmp), L"%x %X", localTime);
-  return UTF16ToUTF8(tmp);
+  return WStringToUTF8(tmp);
 #else
   char tmp[32] = {};
   strftime(tmp, sizeof(tmp), "%x %X", localTime);

Source/Core/Core/HW/WiimoteReal/IOWin.cpp

@@ -556,7 +556,7 @@ void WiimoteScannerWindows::FindWiimotes(std::vector<Wiimote*>& found_wiimotes,
 
       WinWriteMethod write_method = GetInitialWriteMethod(IsUsingToshibaStack);
 
-      if (!IsNewWiimote(UTF16ToUTF8(device_path)) || !IsWiimote(device_path, write_method))
+      if (!IsNewWiimote(WStringToUTF8(device_path)) || !IsWiimote(device_path, write_method))
       {
         free(detail_data);
         continue;
@@ -608,7 +608,7 @@ bool WiimoteWindows::ConnectInternal()
   if (IsConnected())
     return true;
 
-  if (!IsNewWiimote(UTF16ToUTF8(m_devicepath)))
+  if (!IsNewWiimote(WStringToUTF8(m_devicepath)))
     return false;
 
   auto const open_flags = FILE_SHARE_READ | FILE_SHARE_WRITE;
@@ -886,7 +886,7 @@ void ProcessWiimotes(bool new_scan, const T& callback)
         DEBUG_LOG(WIIMOTE, "Authenticated %i connected %i remembered %i ", btdi.fAuthenticated,
                   btdi.fConnected, btdi.fRemembered);
 
-        if (IsValidDeviceName(UTF16ToUTF8(btdi.szName)))
+        if (IsValidDeviceName(WStringToUTF8(btdi.szName)))
         {
           callback(hRadio, radioInfo, btdi);
         }

Source/Core/Core/HW/WiimoteReal/IOWin.h

@@ -26,7 +26,7 @@ class WiimoteWindows final : public Wiimote
 public:
   WiimoteWindows(const std::basic_string<TCHAR>& path, WinWriteMethod initial_write_method);
   ~WiimoteWindows() override;
-  std::string GetId() const override { return UTF16ToUTF8(m_devicepath); }
+  std::string GetId() const override { return WStringToUTF8(m_devicepath); }
 
 protected:
   bool ConnectInternal() override;

Source/Core/Core/HW/WiimoteReal/IOhidapi.cpp

@@ -61,7 +61,7 @@ void WiimoteScannerHidapi::FindWiimotes(std::vector<Wiimote*>& wiimotes, Wiimote
   hid_device_info* list = hid_enumerate(0x0, 0x0);
   for (hid_device_info* device = list; device; device = device->next)
   {
-    const std::string name = device->product_string ? UTF16ToUTF8(device->product_string) : "";
+    const std::string name = device->product_string ? WStringToUTF8(device->product_string) : "";
     const bool is_wiimote =
         IsValidDeviceName(name) || (device->vendor_id == 0x057e &&
                                     (device->product_id == 0x0306 || device->product_id == 0x0330));

Source/Core/DolphinNoGUI/PlatformWin32.cpp

@@ -113,7 +113,7 @@ bool PlatformWin32::Init()
 
 void PlatformWin32::SetTitle(const std::string& string)
 {
-  SetWindowTextW(m_hwnd, UTF8ToUTF16(string).c_str());
+  SetWindowTextW(m_hwnd, UTF8ToWString(string).c_str());
 }
 
 void PlatformWin32::MainLoop()

Source/Core/InputCommon/ControllerInterface/DInput/DInput.cpp

@@ -38,7 +38,7 @@ std::string GetDeviceName(const LPDIRECTINPUTDEVICE8 device)
   std::string result;
   if (SUCCEEDED(device->GetProperty(DIPROP_PRODUCTNAME, &str.diph)))
   {
-    result = StripSpaces(UTF16ToUTF8(str.wsz));
+    result = StripSpaces(WStringToUTF8(str.wsz));
   }
   else
   {

Source/Core/UICommon/AutoUpdate.cpp

@@ -237,7 +237,7 @@ void AutoUpdateChecker::TriggerUpdate(const AutoUpdateChecker::NewVersionInforma
   STARTUPINFO sinfo = {sizeof(sinfo)};
   sinfo.dwFlags = STARTF_FORCEOFFFEEDBACK;  // No hourglass cursor after starting the process.
   PROCESS_INFORMATION pinfo;
-  if (CreateProcessW(UTF8ToUTF16(reloc_updater_path).c_str(), UTF8ToUTF16(command_line).data(),
+  if (CreateProcessW(UTF8ToWString(reloc_updater_path).c_str(), UTF8ToWString(command_line).data(),
                      nullptr, nullptr, FALSE, 0, nullptr, nullptr, &sinfo, &pinfo))
   {
     CloseHandle(pinfo.hThread);

Source/Core/VideoBackends/D3DCommon/Common.cpp

@@ -110,7 +110,7 @@ std::vector<std::string> GetAdapterNames()
     std::string name;
     DXGI_ADAPTER_DESC desc;
     if (SUCCEEDED(adapter->GetDesc(&desc)))
-      name = UTF16ToUTF8(desc.Description);
+      name = WStringToUTF8(desc.Description);
 
     adapters.push_back(std::move(name));
   }

Source/Core/WinUpdater/Main.cpp

@@ -28,7 +28,7 @@ std::vector<std::string> CommandLineToUtf8Argv(PCWSTR command_line)
   std::vector<std::string> argv(nargs);
   for (int i = 0; i < nargs; ++i)
   {
-    argv[i] = UTF16ToUTF8(tokenized[i]);
+    argv[i] = WStringToUTF8(tokenized[i]);
   }
 
   LocalFree(tokenized);

Source/Core/WinUpdater/WinUI.cpp

@@ -180,7 +180,7 @@ void ResetCurrentProgress()
 
 void Error(const std::string& text)
 {
-  auto wide_text = UTF8ToUTF16(text);
+  auto wide_text = UTF8ToWString(text);
 
   MessageBox(nullptr,
              (L"A fatal error occured and the updater cannot continue:\n " + wide_text).c_str(),
@@ -200,7 +200,7 @@ void SetCurrentProgress(int current, int total)
 
 void SetDescription(const std::string& text)
 {
-  SetWindowText(label_handle, UTF8ToUTF16(text).c_str());
+  SetWindowText(label_handle, UTF8ToWString(text).c_str());
 }
 
 void MessageLoop()
@@ -256,7 +256,7 @@ void LaunchApplication(std::string path)
 {
   // Hack: Launching the updater over the explorer ensures that admin priviliges are dropped. Why?
   // Ask Microsoft.
-  ShellExecuteW(nullptr, nullptr, L"explorer.exe", UTF8ToUTF16(path).c_str(), nullptr, SW_SHOW);
+  ShellExecuteW(nullptr, nullptr, L"explorer.exe", UTF8ToWString(path).c_str(), nullptr, SW_SHOW);
 }
 
 void Sleep(int sleep)