Convert VolumeDirectory names back to SHIFT-JIS (issue #9988)

Source/Core/Common/StringUtil.cpp

@@ -25,6 +25,8 @@
 
 #ifdef _WIN32
 #include <Windows.h>
+constexpr u32 CODEPAGE_SHIFT_JIS = 932;
+constexpr u32 CODEPAGE_WINDOWS_1252 = 1252;
 #else
 #include <errno.h>
 #include <iconv.h>
@@ -188,7 +190,7 @@ std::string ArrayToString(const u8* data, u32 size, int line_len, bool spaces)
 
   for (int line = 0; size; ++data, --size)
   {
-    oss << std::setw(2) << (int)*data;
+    oss << std::setw(2) << static_cast<int>(*data);
 
     if (line_len == ++line)
     {
@@ -407,15 +409,15 @@ bool StringEndsWith(const std::string& str, const std::string& end)
 
 std::string UTF16ToUTF8(const std::wstring& input)
 {
-  auto const size = WideCharToMultiByte(CP_UTF8, 0, input.data(), (int)input.size(), nullptr, 0,
+  auto const size = WideCharToMultiByte(CP_UTF8, 0, input.data(), static_cast<int>(input.size()),
-                                        nullptr, nullptr);
+                                        nullptr, 0, nullptr, nullptr);
 
   std::string output;
   output.resize(size);
 
   if (size == 0 ||
-      size != WideCharToMultiByte(CP_UTF8, 0, input.data(), (int)input.size(), &output[0],
+      size != WideCharToMultiByte(CP_UTF8, 0, input.data(), static_cast<int>(input.size()),
-                                  (int)output.size(), nullptr, nullptr))
+                                  &output[0], static_cast<int>(output.size()), nullptr, nullptr))
   {
     output.clear();
   }
@@ -425,14 +427,15 @@ std::string UTF16ToUTF8(const std::wstring& input)
 
 std::wstring CPToUTF16(u32 code_page, const std::string& input)
 {
-  auto const size = MultiByteToWideChar(code_page, 0, input.data(), (int)input.size(), nullptr, 0);
+  auto const size =
+      MultiByteToWideChar(code_page, 0, input.data(), static_cast<int>(input.size()), nullptr, 0);
 
   std::wstring output;
   output.resize(size);
 
   if (size == 0 ||
-      size != MultiByteToWideChar(code_page, 0, input.data(), (int)input.size(), &output[0],
+      size != MultiByteToWideChar(code_page, 0, input.data(), static_cast<int>(input.size()),
-                                  (int)output.size()))
+                                  &output[0], static_cast<int>(output.size())))
   {
     output.clear();
   }
@@ -440,6 +443,25 @@ std::wstring CPToUTF16(u32 code_page, const std::string& input)
   return output;
 }
 
+std::string UTF16ToCP(u32 code_page, const std::wstring& input)
+{
+  auto const size = WideCharToMultiByte(code_page, 0, input.data(), static_cast<int>(input.size()),
+                                        nullptr, 0, nullptr, false);
+
+  std::string output;
+  output.resize(size);
+
+  if (size == 0 ||
+      size != WideCharToMultiByte(code_page, 0, input.data(), static_cast<int>(input.size()),
+                                  &output[0], static_cast<int>(output.size()), nullptr, false))
+  {
+    const DWORD error_code = GetLastError();
+    ERROR_LOG(COMMON, "WideCharToMultiByte Error in String '%s': %lu", input.c_str(), error_code);
+    output.clear();
+  }
+  return output;
+}
+
 std::wstring UTF8ToUTF16(const std::string& input)
 {
   return CPToUTF16(CP_UTF8, input);
@@ -447,22 +469,27 @@ std::wstring UTF8ToUTF16(const std::string& input)
 
 std::string SHIFTJISToUTF8(const std::string& input)
 {
-  return UTF16ToUTF8(CPToUTF16(932, input));
+  return UTF16ToUTF8(CPToUTF16(CODEPAGE_SHIFT_JIS, input));
+}
+
+std::string UTF8ToSHIFTJIS(const std::string& input)
+{
+  return UTF16ToCP(CODEPAGE_SHIFT_JIS, UTF8ToUTF16(input));
 }
 
 std::string CP1252ToUTF8(const std::string& input)
 {
-  return UTF16ToUTF8(CPToUTF16(1252, input));
+  return UTF16ToUTF8(CPToUTF16(CODEPAGE_WINDOWS_1252, input));
 }
 
 #else
 
 template <typename T>
-std::string CodeToUTF8(const char* fromcode, const std::basic_string<T>& input)
+std::string CodeTo(const char* tocode, const char* fromcode, const std::basic_string<T>& input)
 {
   std::string result;
 
-  iconv_t const conv_desc = iconv_open("UTF-8", fromcode);
+  iconv_t const conv_desc = iconv_open(tocode, fromcode);
   if ((iconv_t)-1 == conv_desc)
   {
     ERROR_LOG(COMMON, "Iconv initialization failure [%s]: %s", fromcode, strerror(errno));
@@ -513,6 +540,12 @@ std::string CodeToUTF8(const char* fromcode, const std::basic_string<T>& input)
   return result;
 }
 
+template <typename T>
+std::string CodeToUTF8(const char* fromcode, const std::basic_string<T>& input)
+{
+  return CodeTo("UTF-8", fromcode, input);
+}
+
 std::string CP1252ToUTF8(const std::string& input)
 {
   // return CodeToUTF8("CP1252//TRANSLIT", input);
@@ -526,6 +559,11 @@ std::string SHIFTJISToUTF8(const std::string& input)
   return CodeToUTF8("SJIS", input);
 }
 
+std::string UTF8ToSHIFTJIS(const std::string& input)
+{
+  return CodeTo("SJIS", "UTF-8", input);
+}
+
 std::string UTF16ToUTF8(const std::wstring& input)
 {
   std::string result = CodeToUTF8("UTF-16LE", input);

Source/Core/Common/StringUtil.h

@@ -122,6 +122,7 @@ bool StringEndsWith(const std::string& str, const std::string& end);
 
 std::string CP1252ToUTF8(const std::string& str);
 std::string SHIFTJISToUTF8(const std::string& str);
+std::string UTF8ToSHIFTJIS(const std::string& str);
 std::string UTF16ToUTF8(const std::wstring& str);
 
 #ifdef _WIN32

Source/Core/DiscIO/VolumeDirectory.cpp

@@ -17,6 +17,7 @@
 #include "Common/CommonTypes.h"
 #include "Common/FileUtil.h"
 #include "Common/Logging/Log.h"
+#include "Common/StringUtil.h"
 #include "DiscIO/Blob.h"
 #include "DiscIO/Enums.h"
 #include "DiscIO/Volume.h"
@@ -26,6 +27,7 @@ namespace DiscIO
 {
 static u32 ComputeNameSize(const File::FSTEntry& parent_entry);
 static std::string ASCIIToUppercase(std::string str);
+static void ConvertUTF8NamesToSHIFTJIS(File::FSTEntry& parent_entry);
 
 const size_t CVolumeDirectory::MAX_NAME_LENGTH;
 const size_t CVolumeDirectory::MAX_ID_LENGTH;
@@ -350,6 +352,9 @@ void CVolumeDirectory::BuildFST()
   m_fst_data.clear();
 
   File::FSTEntry rootEntry = File::ScanDirectoryTree(m_root_directory, true);
+
+  ConvertUTF8NamesToSHIFTJIS(rootEntry);
+
   u32 name_table_size = Common::AlignUp(ComputeNameSize(rootEntry), 1ull << m_address_shift);
   u64 total_entries = rootEntry.size + 1;  // The root entry itself isn't counted in rootEntry.size
 
@@ -501,6 +506,17 @@ static u32 ComputeNameSize(const File::FSTEntry& parent_entry)
   return name_size;
 }
 
+static void ConvertUTF8NamesToSHIFTJIS(File::FSTEntry& parent_entry)
+{
+  for (File::FSTEntry& entry : parent_entry.children)
+  {
+    if (entry.isDirectory)
+      ConvertUTF8NamesToSHIFTJIS(entry);
+
+    entry.virtualName = UTF8ToSHIFTJIS(entry.virtualName);
+  }
+}
+
 static std::string ASCIIToUppercase(std::string str)
 {
   std::transform(str.begin(), str.end(), str.begin(),

Source/UnitTests/Common/StringUtilTest.cpp

@@ -40,3 +40,12 @@ TEST(StringUtil, StringEndsWith)
   EXPECT_TRUE(StringEndsWith("abc", ""));
   EXPECT_TRUE(StringEndsWith("", ""));
 }
+
+TEST(StringUtil, UTF8ToSHIFTJIS)
+{
+  const std::string kirby_unicode = "\xe6\x98\x9f\xe3\x81\xae\xe3\x82\xab\xe3\x83\xbc\xe3\x83\x93\xe3\x82\xa3";
+  const std::string kirby_sjis = "\x90\xaf\x82\xcc\x83\x4a\x81\x5b\x83\x72\x83\x42";
+  
+  EXPECT_STREQ(SHIFTJISToUTF8(UTF8ToSHIFTJIS(kirby_unicode)).c_str(), kirby_unicode.c_str());
+  EXPECT_STREQ(UTF8ToSHIFTJIS(kirby_unicode).c_str(), kirby_sjis.c_str());
+}