fix regressions in handling of weird characters in filenames/archives, dating from introduction of fex. also improve fex to support utf-8 extended filenames in zipfiles.

desmume/src/windows/DeSmuME_2005.vcproj

@@ -988,6 +988,10 @@
 					RelativePath=".\File_Extractor\fex\Zip_Extractor.cpp"
 					>
 				</File>
+				<File
+					RelativePath=".\File_Extractor\fex\Zip_Extractor.h"
+					>
+				</File>
 				<File
 					RelativePath=".\File_Extractor\fex\Zlib_Inflater.cpp"
 					>

desmume/src/windows/FEX_Interface.cpp

@@ -129,8 +129,18 @@ ArchiveFile::ArchiveFile(const char* filename)
 	if(!file)
 		return;
 
-	m_filename = new char[strlen(filename)+1];
+	//TODO - maybe windows only check here if we ever drop fex into portable code. unfortunately this is probably too unwieldy to ever happen
-	strcpy(m_filename, filename);
+
+	//convert the filename to unicode
+	wchar_t temp_wchar[MAX_PATH*2];
+	char filename_utf8[MAX_PATH*4];
+	MultiByteToWideChar(CP_THREAD_ACP,0,filename,-1,temp_wchar,ARRAY_SIZE(temp_wchar));
+	//now convert it back to utf-8. is there a way to do this in one step?
+	WideCharToMultiByte(CP_UTF8,0,temp_wchar,-1,filename_utf8,ARRAY_SIZE(filename_utf8), NULL, NULL);
+
+
+	m_filename = new char[strlen(filename_utf8)+1];
+	strcpy(m_filename, filename_utf8);
 
 	// detect archive type using format signature in file
 	for(size_t i = 0; i < s_formatInfos.size() && m_typeIndex < 0; i++)
@@ -187,11 +197,13 @@ ArchiveFile::ArchiveFile(const char* filename)
 
 		m_items[0].name = new char[strlen(filename)+1];
 		strcpy(m_items[0].name, filename);
+
+		m_items[0].wname = _wcsdup(temp_wchar);
 	}
 	else
 	{
 		fex_t * object;
-		fex_err_t err = fex_open_type( &object, filename, s_formatInfos[m_typeIndex].type );
+		fex_err_t err = fex_open_type( &object, m_filename, s_formatInfos[m_typeIndex].type );
 		if ( !err )
 		{
 			int numItems = 0;
@@ -219,6 +231,21 @@ ArchiveFile::ArchiveFile(const char* filename)
 				item.name = new char[strlen(name)+1];
 				strcpy(item.name, name);
 
+				const wchar_t* wname = fex_wname(object);
+				if(wname)
+				{
+					item.wname = _wcsdup(fex_wname(object));
+				}
+				else
+				{
+					const char* name = fex_name(object);
+					wchar_t temp_wchar[MAX_PATH];
+					//what code page to use??? who knows. 
+					MultiByteToWideChar(CP_ACP,0,name,-1,temp_wchar,ARRAY_SIZE(temp_wchar));
+					item.wname = _wcsdup(temp_wchar);
+				}
+
+
 				err = fex_stat(object);
 				if (err) break;
 
@@ -240,6 +267,7 @@ ArchiveFile::~ArchiveFile()
 	for(int i = 0; i < m_numItems; i++)
 	{
 		delete[] m_items[i].name;
+		free(m_items[i].wname);
 	}
 	delete[] m_items;
 	delete[] m_filename;
@@ -274,6 +302,14 @@ const char* ArchiveFile::GetItemName(int item)
 	return m_items[item].name;
 }
 
+const wchar_t* ArchiveFile::GetItemNameW(int item)
+{
+	//assert(item >= 0 && item < m_numItems);
+	if(!(item >= 0 && item < m_numItems)) return L"";
+	return m_items[item].wname;
+}
+
+
 bool ArchiveFile::IsCompressed()
 {
 	return (m_typeIndex >= 0);

desmume/src/windows/FEX_Interface.h

@@ -39,6 +39,7 @@ struct ArchiveFile
 	int GetNumItems();
 	int GetItemSize(int item);
 	const char* GetItemName(int item);
+	const wchar_t* GetItemNameW(int item);
 	int ExtractItem(int item, unsigned char* outBuffer, int bufSize) const; // returns size, or 0 if failed
 	int ExtractItem(int item, const char* outFilename) const;
 
@@ -50,6 +51,7 @@ protected:
 	{
 		int size;
 		char* name;
+		wchar_t* wname;
 		s64 offset;
 	};
 	ArchiveItem* m_items;

desmume/src/windows/File_Extractor/fex/Zip_Extractor.cpp

@@ -34,6 +34,8 @@ int const disk_block_size = 4 * 1024;
 // Read buffer used for extracting file data
 int const read_buf_size = 16 * 1024;
 
+#include <string>
+
 struct header_t
 {
 	char type [4];
@@ -250,12 +252,64 @@ blargg_err_t Zip_Extractor::update_info( bool advance_first )
 
 		if ( !advance_first )
 		{
+			char unterminate = e.filename[len];
 			e.filename [len] = 0; // terminate name
+			std::string fname = e.filename;
 			
 			if ( is_normal_file( e, len ) )
 			{
-				set_name( e.filename );
+				e.filename[len] = unterminate;
+				name.resize(fname.size()+1);
+				if(len != 0)
+				{
+					memcpy(name.begin(),fname.c_str(),len);
+					name[name.size()-1] = 0;
+				}
+				set_name( name.begin() );
 				set_info( get_le32( e.size ), get_le32( e.date ), get_le32( e.crc ) );
+
+				unsigned extra_len = get_le32(e.extra_len);
+
+				//walk over extra fields
+				unsigned i = len;
+				while(i < extra_len + len)
+				{
+					unsigned id = get_le16(e.filename + i);
+					i += 2;
+					unsigned exlen = get_le16(e.filename + i);
+					i += 2;
+					int exfield = i;
+					i += exlen;
+					if(id == 0x7075) //INFO-ZIP unicode path extra field (contains version, checksum, and utf-8 filename)
+					{
+						unsigned version = (unsigned char)*(e.filename + exfield);
+						if(version == 1)
+						{
+							exfield += 1; //skip version
+							exfield += 4; //skip crc
+							//the remainder is a utf-8 filename
+							int fnamelen = exlen-5;
+							char* tempbuf = (char*)malloc(fnamelen + 1);
+							memcpy(tempbuf,e.filename + exfield, fnamelen);
+							tempbuf[fnamelen] = 0;
+							wchar_t* wfname_buf = blargg_to_wide(tempbuf);
+							std::wstring wfname = wfname_buf;
+							free(tempbuf);
+							free(wfname_buf);
+							
+							size_t wfname_len = wfname.size();
+
+							this->wname.resize(wfname_len+1);
+							if(wfname_len != 0)
+							{
+								memcpy(this->wname.begin(),wfname.c_str(),wfname_len*sizeof(wchar_t));
+								wname[wname.size()-1] = 0;
+							}
+							set_name( name.begin(), wname.begin() );
+							
+						}
+					}
+				}
 				break;
 			}
 		}

desmume/src/windows/File_Extractor/fex/Zip_Extractor.h

@@ -40,6 +40,9 @@ private:
 	blargg_err_t first_read( int count );
 	void reorder_entry_header( int offset );
 	static blargg_err_t inflater_read( void* data, void* out, int* count );
+
+	blargg_vector<char> name;
+	blargg_vector<wchar_t> wname;
 };
 
 #endif

desmume/src/windows/OpenArchive.cpp

@@ -51,7 +51,10 @@ tryagain:
 		{
 			if(archive.GetItemSize(i))
 			{
-				const char* name = archive.GetItemName(i);
+				//bullshit time. convert to system locale
+				char name[MAX_PATH];
+				WideCharToMultiByte(CP_ACP,0,archive.GetItemNameW(i),-1,name,ARRAY_SIZE(name), NULL, NULL);
+
 				const char* ext = strrchr(name, '.');
 				bool ok = true;
 				if(ext++)
@@ -454,7 +457,15 @@ bool ObtainFile(const char* Name, char *const & LogicalName, char *const & Physi
 				s_tempFiles.ReleaseFile(TempFileName);
 			s_tempFiles.ReleaseFile(PhysicalName);
 			strcpy(PhysicalName, TempFileName);
-			_snprintf(LogicalName + strlen(LogicalName), 1024 - (strlen(LogicalName)+1), "|%s", archive.GetItemName(item));
+
+			const wchar_t* itemNameW = archive.GetItemNameW(item);
+
+			//convert the itemname to local encoding
+			char itemname_utf8[MAX_PATH*4];
+			WideCharToMultiByte(CP_THREAD_ACP,0,itemNameW,-1,itemname_utf8,ARRAY_SIZE(itemname_utf8),NULL,NULL);
+
+			//strcat(LogicalName,itemname_utf8);
+			_snprintf(LogicalName + strlen(LogicalName), 1024 - (strlen(LogicalName)+1), "|%s", itemname_utf8);
 		}
 	}
 }