FileSystem: Handle paths longer than MAX_PATH on Windows

common/CMakeLists.txt

@@ -136,7 +136,7 @@ endif()
 if(WIN32)
 	enable_language(ASM_MASM)
 	target_sources(common PRIVATE FastJmp.asm)
-	target_link_libraries(common PUBLIC WIL::WIL winmm)
+	target_link_libraries(common PUBLIC WIL::WIL winmm pathcch)
 	target_sources(common PRIVATE
 		CrashHandler.cpp
 		CrashHandler.h

common/CrashHandler.cpp

@@ -180,7 +180,7 @@ void CrashHandler::SetWriteDirectory(const std::string_view& dump_directory)
 	if (!s_veh_handle)
 		return;
 
-	s_write_directory = StringUtil::UTF8StringToWideString(dump_directory);
+	s_write_directory = FileSystem::GetWin32Path(dump_directory);
 }
 
 void CrashHandler::WriteDumpForCaller()

common/FileSystem.cpp

@@ -1,4 +1,4 @@
-// SPDX-FileCopyrightText: 2002-2023 PCSX2 Dev Team
+// SPDX-FileCopyrightText: 2002-2024 PCSX2 Dev Team
 // SPDX-License-Identifier: LGPL-3.0+
 
 #include "FileSystem.h"
@@ -28,6 +28,8 @@
 #if defined(_WIN32)
 #include "RedtapeWindows.h"
 #include <io.h>
+#include <malloc.h>
+#include <pathcch.h>
 #include <winioctl.h>
 #include <share.h>
 #include <shlobj.h>
@@ -42,6 +44,7 @@
 #endif
 
 #ifdef _WIN32
+
 static std::time_t ConvertFileTimeToUnixTime(const FILETIME& ft)
 {
 	// based off https://stackoverflow.com/a/6161842
@@ -51,6 +54,13 @@ static std::time_t ConvertFileTimeToUnixTime(const FILETIME& ft)
 	const s64 full = static_cast<s64>((static_cast<u64>(ft.dwHighDateTime) << 32) | static_cast<u64>(ft.dwLowDateTime));
 	return static_cast<std::time_t>(full / WINDOWS_TICK - SEC_TO_UNIX_EPOCH);
 }
+
+template <class T>
+static bool IsUNCPath(const T& path)
+{
+	return (path.length() >= 3 && path[0] == '\\' && path[1] == '\\');
+}
+
 #endif
 
 static inline bool FileSystemCharacterIsSane(char32_t c, bool strip_slashes)
@@ -201,6 +211,66 @@ bool Path::IsValidFileName(const std::string_view& str, bool allow_slashes)
 	return true;
 }
 
+#ifdef _WIN32
+
+bool FileSystem::GetWin32Path(std::wstring* dest, std::string_view str)
+{
+	// Just convert to wide if it's a relative path, MAX_PATH still applies.
+	if (!Path::IsAbsolute(str))
+		return StringUtil::UTF8StringToWideString(*dest, str);
+
+	// PathCchCanonicalizeEx() thankfully takes care of everything.
+	// But need to widen the string first, avoid the stack allocation.
+	int wlen = MultiByteToWideChar(CP_UTF8, 0, str.data(), static_cast<int>(str.length()), nullptr, 0);
+	if (wlen <= 0) [[unlikely]]
+		return false;
+
+	// So copy it to a temp wide buffer first.
+	wchar_t* wstr_buf = static_cast<wchar_t*>(_malloca(sizeof(wchar_t) * (static_cast<size_t>(wlen) + 1)));
+	wlen = MultiByteToWideChar(CP_UTF8, 0, str.data(), static_cast<int>(str.length()), wstr_buf, wlen);
+	if (wlen <= 0) [[unlikely]]
+	{
+		_freea(wstr_buf);
+		return false;
+	}
+
+	// And use PathCchCanonicalizeEx() to fix up any non-direct elements.
+	wstr_buf[wlen] = '\0';
+	dest->resize(std::max<size_t>(static_cast<size_t>(wlen) + (IsUNCPath(str) ? 9 : 5), 16));
+	for (;;)
+	{
+		const HRESULT hr =
+			PathCchCanonicalizeEx(dest->data(), dest->size(), wstr_buf, PATHCCH_ENSURE_IS_EXTENDED_LENGTH_PATH);
+		if (SUCCEEDED(hr))
+		{
+			dest->resize(std::wcslen(dest->data()));
+			_freea(wstr_buf);
+			return true;
+		}
+		else if (hr == HRESULT_FROM_WIN32(ERROR_INSUFFICIENT_BUFFER))
+		{
+			dest->resize(dest->size() * 2);
+			continue;
+		}
+		else [[unlikely]]
+		{
+			Console.ErrorFmt("PathCchCanonicalizeEx() returned {:08X}", static_cast<unsigned>(hr));
+			_freea(wstr_buf);
+			return false;
+		}
+	}
+}
+
+std::wstring FileSystem::GetWin32Path(std::string_view str)
+{
+	std::wstring ret;
+	if (!GetWin32Path(&ret, str))
+		ret.clear();
+	return ret;
+}
+
+#endif
+
 bool Path::IsAbsolute(const std::string_view& path)
 {
 #ifdef _WIN32
@@ -237,16 +307,28 @@ std::string Path::RealPath(const std::string_view& path)
 	symlink_buf.resize(path.size() + 1);
 
 	// Check for any symbolic links throughout the path while adding components.
+	const bool skip_first = IsUNCPath(path);
 	bool test_symlink = true;
 	for (const std::string_view& comp : components)
 	{
 		if (!realpath.empty())
+		{
 			realpath.push_back(FS_OSPATH_SEPARATOR_CHARACTER);
 			realpath.append(comp);
+		}
+		else if (skip_first)
+		{
+			realpath.append(comp);
+			continue;
+		}
+		else
+		{
+			realpath.append(comp);
+		}
 		if (test_symlink)
 		{
 			DWORD attribs;
-			if (StringUtil::UTF8StringToWideString(wrealpath, realpath) &&
+			if (FileSystem::GetWin32Path(&wrealpath, realpath) &&
 				(attribs = GetFileAttributesW(wrealpath.c_str())) != INVALID_FILE_ATTRIBUTES)
 			{
 				// if not a link, go to the next component
@@ -285,7 +367,14 @@ std::string Path::RealPath(const std::string_view& path)
 
 	// GetFinalPathNameByHandleW() adds a \\?\ prefix, so remove it.
 	if (realpath.starts_with("\\\\?\\") && IsAbsolute(std::string_view(realpath.data() + 4, realpath.size() - 4)))
+	{
 		realpath.erase(0, 4);
+	}
+	else if (realpath.starts_with("\\\\?\\UNC\\"))
+	{
+		realpath.erase(0, 7);
+		realpath.insert(realpath.begin(), '\\');
+	}
 
 #else
 	// Why this monstrosity instead of calling realpath()? realpath() only works on files that exist.
@@ -871,8 +960,8 @@ std::string Path::CreateFileURL(std::string_view path)
 std::FILE* FileSystem::OpenCFile(const char* filename, const char* mode, Error* error)
 {
 #ifdef _WIN32
-	const std::wstring wfilename(StringUtil::UTF8StringToWideString(filename));
+	const std::wstring wfilename = GetWin32Path(filename);
-	const std::wstring wmode(StringUtil::UTF8StringToWideString(mode));
+	const std::wstring wmode = GetWin32Path(mode);
 	if (!wfilename.empty() && !wmode.empty())
 	{
 		std::FILE* fp;
@@ -906,7 +995,7 @@ std::FILE* FileSystem::OpenCFile(const char* filename, const char* mode, Error*
 int FileSystem::OpenFDFile(const char* filename, int flags, int mode, Error* error)
 {
 #ifdef _WIN32
-	const std::wstring wfilename(StringUtil::UTF8StringToWideString(filename));
+	const std::wstring wfilename = GetWin32Path(filename);
 	if (!wfilename.empty())
 		return _wopen(wfilename.c_str(), flags, mode);
 
@@ -927,8 +1016,8 @@ FileSystem::ManagedCFilePtr FileSystem::OpenManagedCFile(const char* filename, c
 std::FILE* FileSystem::OpenSharedCFile(const char* filename, const char* mode, FileShareMode share_mode, Error* error)
 {
 #ifdef _WIN32
-	const std::wstring wfilename(StringUtil::UTF8StringToWideString(filename));
+	const std::wstring wfilename = GetWin32Path(filename);
-	const std::wstring wmode(StringUtil::UTF8StringToWideString(mode));
+	const std::wstring wmode = GetWin32Path(mode);
 	if (wfilename.empty() || wmode.empty())
 		return nullptr;
 
@@ -1163,8 +1252,7 @@ bool FileSystem::CopyFilePath(const char* source, const char* destination, bool
 
 	return true;
 #else
-	return CopyFileW(StringUtil::UTF8StringToWideString(source).c_str(),
+	return CopyFileW(GetWin32Path(source).c_str(), GetWin32Path(destination).c_str(), !replace);
-		StringUtil::UTF8StringToWideString(destination).c_str(), !replace);
 #endif
 }
 
@@ -1205,7 +1293,7 @@ static u32 RecursiveFindFiles(const char* origin_path, const char* parent_path,
 	std::string utf8_filename;
 	utf8_filename.reserve((sizeof(wfd.cFileName) / sizeof(wfd.cFileName[0])) * 2);
 
-	const HANDLE hFind = FindFirstFileW(StringUtil::UTF8StringToWideString(search_dir).c_str(), &wfd);
+	const HANDLE hFind = FindFirstFileW(FileSystem::GetWin32Path(search_dir).c_str(), &wfd);
 	if (hFind == INVALID_HANDLE_VALUE)
 		return 0;
 
@@ -1370,7 +1458,7 @@ bool FileSystem::StatFile(const char* path, struct stat* st)
 		return false;
 
 	// convert to wide string
-	const std::wstring wpath(StringUtil::UTF8StringToWideString(path));
+	const std::wstring wpath = GetWin32Path(path);
 	if (wpath.empty())
 		return false;
 
@@ -1403,7 +1491,7 @@ bool FileSystem::StatFile(const char* path, FILESYSTEM_STAT_DATA* sd)
 		return false;
 
 	// convert to wide string
-	const std::wstring wpath(StringUtil::UTF8StringToWideString(path));
+	const std::wstring wpath = GetWin32Path(path);
 	if (wpath.empty())
 		return false;
 
@@ -1481,7 +1569,7 @@ bool FileSystem::FileExists(const char* path)
 		return false;
 
 	// convert to wide string
-	const std::wstring wpath(StringUtil::UTF8StringToWideString(path));
+	const std::wstring wpath = GetWin32Path(path);
 	if (wpath.empty())
 		return false;
 
@@ -1503,7 +1591,7 @@ bool FileSystem::DirectoryExists(const char* path)
 		return false;
 
 	// convert to wide string
-	const std::wstring wpath(StringUtil::UTF8StringToWideString(path));
+	const std::wstring wpath = GetWin32Path(path);
 	if (wpath.empty())
 		return false;
 
@@ -1520,7 +1608,7 @@ bool FileSystem::DirectoryExists(const char* path)
 
 bool FileSystem::DirectoryIsEmpty(const char* path)
 {
-	std::wstring wpath(StringUtil::UTF8StringToWideString(path));
+	std::wstring wpath = GetWin32Path(path);
 	wpath += L"\\*";
 
 	WIN32_FIND_DATAW wfd;
@@ -1547,7 +1635,7 @@ bool FileSystem::DirectoryIsEmpty(const char* path)
 
 bool FileSystem::CreateDirectoryPath(const char* Path, bool Recursive)
 {
-	const std::wstring wpath(StringUtil::UTF8StringToWideString(Path));
+	const std::wstring wpath = GetWin32Path(Path);
 
 	// has a path
 	if (wpath.empty())
@@ -1628,7 +1716,7 @@ bool FileSystem::DeleteFilePath(const char* path)
 	if (path[0] == '\0')
 		return false;
 
-	const std::wstring wpath(StringUtil::UTF8StringToWideString(path));
+	const std::wstring wpath = GetWin32Path(path);
 	const DWORD fileAttributes = GetFileAttributesW(wpath.c_str());
 	if (fileAttributes == INVALID_FILE_ATTRIBUTES || fileAttributes & FILE_ATTRIBUTE_DIRECTORY)
 		return false;
@@ -1638,8 +1726,8 @@ bool FileSystem::DeleteFilePath(const char* path)
 
 bool FileSystem::RenamePath(const char* old_path, const char* new_path)
 {
-	const std::wstring old_wpath(StringUtil::UTF8StringToWideString(old_path));
+	const std::wstring old_wpath = GetWin32Path(old_path);
-	const std::wstring new_wpath(StringUtil::UTF8StringToWideString(new_path));
+	const std::wstring new_wpath = GetWin32Path(new_path);
 
 	if (!MoveFileExW(old_wpath.c_str(), new_wpath.c_str(), MOVEFILE_REPLACE_EXISTING))
 	{
@@ -1652,7 +1740,7 @@ bool FileSystem::RenamePath(const char* old_path, const char* new_path)
 
 bool FileSystem::DeleteDirectory(const char* path)
 {
-	const std::wstring wpath(StringUtil::UTF8StringToWideString(path));
+	const std::wstring wpath = GetWin32Path(path);
 	return RemoveDirectoryW(wpath.c_str());
 }
 
@@ -1700,13 +1788,13 @@ std::string FileSystem::GetWorkingDirectory()
 
 bool FileSystem::SetWorkingDirectory(const char* path)
 {
-	const std::wstring wpath(StringUtil::UTF8StringToWideString(path));
+	const std::wstring wpath = GetWin32Path(path);
 	return (SetCurrentDirectoryW(wpath.c_str()) == TRUE);
 }
 
 bool FileSystem::SetPathCompression(const char* path, bool enable)
 {
-	const std::wstring wpath(StringUtil::UTF8StringToWideString(path));
+	const std::wstring wpath = GetWin32Path(path);
 	const DWORD attrs = GetFileAttributesW(wpath.c_str());
 	if (attrs == INVALID_FILE_ATTRIBUTES)
 		return false;

common/FileSystem.h

@@ -1,4 +1,4 @@
-// SPDX-FileCopyrightText: 2002-2023 PCSX2 Dev Team
+// SPDX-FileCopyrightText: 2002-2024 PCSX2 Dev Team
 // SPDX-License-Identifier: LGPL-3.0+
 
 #pragma once
@@ -163,6 +163,12 @@ namespace FileSystem
 	/// Does nothing and returns false on non-Windows platforms.
 	bool SetPathCompression(const char* path, bool enable);
 
+#ifdef _WIN32
+	// Path limit remover, but also converts to a wide string at the same time.
+	bool GetWin32Path(std::wstring* dest, std::string_view str);
+	std::wstring GetWin32Path(std::string_view str);
+#endif
+
 	/// Abstracts a POSIX file lock.
 #ifndef _WIN32
 	class POSIXLock

common/Windows/WinMisc.cpp

@@ -3,6 +3,7 @@
 
 #if defined(_WIN32)
 
+#include "common/FileSystem.h"
 #include "common/HostSys.h"
 #include "common/RedtapeWindows.h"
 #include "common/StringUtil.h"
@@ -91,7 +92,7 @@ bool WindowInfo::InhibitScreensaver(const WindowInfo& wi, bool inhibit)
 
 bool Common::PlaySoundAsync(const char* path)
 {
-	const std::wstring wpath(StringUtil::UTF8StringToWideString(path));
+	const std::wstring wpath = FileSystem::GetWin32Path(path);
 	return PlaySoundW(wpath.c_str(), NULL, SND_ASYNC | SND_NODEFAULT);
 }
 

pcsx2/CDVD/GzippedFileReader.cpp

@@ -196,7 +196,7 @@ void GzippedFileReader::AsyncPrefetchReset()
 void GzippedFileReader::AsyncPrefetchOpen()
 {
 	hOverlappedFile = CreateFile(
-		StringUtil::UTF8StringToWideString(m_filename).c_str(),
+		FileSystem::GetWin32Path(m_filename).c_str(),
 		GENERIC_READ,
 		FILE_SHARE_READ,
 		NULL,

pcsx2/DEV9/ATA/ATA_State.cpp

@@ -3,7 +3,6 @@
 
 #include "common/Assertions.h"
 #include "common/FileSystem.h"
-#include "common/StringUtil.h"
 
 #include "ATA.h"
 #include "DEV9/DEV9.h"
@@ -117,7 +116,7 @@ void ATA::InitSparseSupport(const std::string& hddPath)
 #ifdef _WIN32
 	hddSparse = false;
 
-	const std::wstring wHddPath(StringUtil::UTF8StringToWideString(hddPath));
+	const std::wstring wHddPath = FileSystem::GetWin32Path(hddPath);
 	const DWORD fileAttributes = GetFileAttributes(wHddPath.c_str());
 	hddSparse = fileAttributes & FILE_ATTRIBUTE_SPARSE_FILE;
 

pcsx2/windows/FlatFileReaderWindows.cpp

@@ -3,7 +3,7 @@
 
 #include "AsyncFileReader.h"
 
-#include "common/StringUtil.h"
+#include "common/FileSystem.h"
 #include "common/Error.h"
 
 FlatFileReader::FlatFileReader(bool shareWrite) : shareWrite(shareWrite)
@@ -30,7 +30,7 @@ bool FlatFileReader::Open(std::string filename, Error* error)
 		shareMode |= FILE_SHARE_WRITE;
 
 	hOverlappedFile = CreateFile(
-		StringUtil::UTF8StringToWideString(m_filename).c_str(),
+		FileSystem::GetWin32Path(m_filename).c_str(),
 		GENERIC_READ,
 		shareMode,
 		NULL,

updater/Updater.cpp

@@ -82,7 +82,7 @@ bool Updater::OpenUpdateZip(const char* path)
 	LookToRead2_Init(&m_look_stream);
 
 #ifdef _WIN32
-	WRes wres = InFile_OpenW(&m_archive_stream.file, StringUtil::UTF8StringToWideString(path).c_str());
+	WRes wres = InFile_OpenW(&m_archive_stream.file, FileSystem::GetWin32Path(path).c_str());
 #else
 	WRes wres = InFile_Open(&m_archive_stream.file, path);
 #endif
@@ -137,7 +137,7 @@ bool Updater::RecursiveDeleteDirectory(const char* path)
 {
 #ifdef _WIN32
 	// making this safer on Win32...
-	std::wstring wpath(StringUtil::UTF8StringToWideString(path));
+	std::wstring wpath = FileSystem::GetWin32Path(path);
 	wpath += L'\0';
 
 	SHFILEOPSTRUCTW op = {};
@@ -353,8 +353,8 @@ bool Updater::CommitUpdate()
 		m_progress->DisplayFormattedInformation("Moving '%s' to '%s'", staging_file_name.c_str(), dest_file_name.c_str());
 #ifdef _WIN32
 		const bool result =
-			MoveFileExW(StringUtil::UTF8StringToWideString(staging_file_name).c_str(),
+			MoveFileExW(FileSystem::GetWin32Path(staging_file_name).c_str(),
-				StringUtil::UTF8StringToWideString(dest_file_name).c_str(), MOVEFILE_REPLACE_EXISTING);
+				FileSystem::GetWin32Path(dest_file_name).c_str(), MOVEFILE_REPLACE_EXISTING);
 #else
 		const bool result = (rename(staging_file_name.c_str(), dest_file_name.c_str()) == 0);
 #endif

updater/UpdaterExtractor.h

@@ -55,7 +55,7 @@ static inline bool ExtractUpdater(const char* archive_path, const char* destinat
 	});
 
 #ifdef _WIN32
-	WRes wres = InFile_OpenW(&instream.file, StringUtil::UTF8StringToWideString(archive_path).c_str());
+	WRes wres = InFile_OpenW(&instream.file, FileSystem::GetWin32Path(archive_path).c_str());
 #else
 	WRes wres = InFile_Open(&instream.file, archive_path);
 #endif

updater/Windows/WindowsUpdater.cpp

@@ -493,8 +493,9 @@ int WINAPI wWinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, LPWSTR lpCmdLi
 	{
 		const std::string full_path = destination_directory + FS_OSPATH_SEPARATOR_STR + actual_exe;
 		progress.DisplayFormattedInformation("Moving '%s' to '%S'", full_path.c_str(), program_to_launch.c_str());
-		const bool ok = MoveFileExW(StringUtil::UTF8StringToWideString(full_path).c_str(),
+		const bool ok = MoveFileExW(FileSystem::GetWin32Path(full_path).c_str(),
-			program_to_launch.c_str(), MOVEFILE_REPLACE_EXISTING);
+			FileSystem::GetWin32Path(StringUtil::WideStringToUTF8String(program_to_launch)).c_str(),
+			MOVEFILE_REPLACE_EXISTING);
 		if (!ok)
 		{
 			progress.DisplayFormattedModalError("Failed to rename '%s' to %S", full_path.c_str(), program_to_launch.c_str());