ShuriZma
/
pcsx2
mirror of https://github.com/PCSX2/pcsx2.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

DEV9: Allow sparse writing to HDD file

This commit is contained in:
TheLastRar 2021-05-18 22:43:18 +01:00 committed by refractionpcsx2
parent 887a1685dd
commit 31c045fdb5
3 changed files with 401 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
pcsx2/DEV9/ATA/ATA.h
View File

@ -21,7 +21,9 @@
#include <mutex>
#include <condition_variable>
#include "common/RedtapeWindows.h"
#include "common/Path.h"
#include "DEV9/SimpleQueue.h"
class ATA
@ -37,6 +39,18 @@ private:
std::FILE* hddImage = nullptr;
u64 hddImageSize;
bool hddSparse = false;
u64 hddSparseBlockSize;
u64 HddSparseStart;
std::unique_ptr<u8[]> hddSparseBlock;
bool hddSparseBlockValid = false;
#ifdef _WIN32
HANDLE hddNativeHandle = INVALID_HANDLE_VALUE;
#elif defined(__POSIX__)
int hddNativeHandle = -1;
#endif
int pioMode;
int sdmaMode;
int mdmaMode;
@ -171,6 +185,8 @@ public:
//ATAwritePIO;
private:
void InitSparseSupport(const std::string& hddPath);
//Info
void CreateHDDinfo(u64 sizeSectors);
void CreateHDDinfoCsum();
@ -203,6 +219,10 @@ private:
void IO_Thread();
void IO_Read();
bool IO_Write();
bool IO_SparseZero(u64 byteOffset, u64 byteSize);
void IO_SparseCacheUpdateLocation(u64 Offset);
void IO_SparseCacheLoad();
bool IsAllZero(const void* data, size_t len);
void HDD_ReadAsync(void (ATA::*drqCMD)());
void HDD_ReadSync(void (ATA::*drqCMD)());
bool HDD_CanAssessOrSetError();

159
pcsx2/DEV9/ATA/ATA_State.cpp
View File

@ -17,6 +17,7 @@
#include "common/Assertions.h"
#include "common/FileSystem.h"
#include "common/StringUtil.h"
#include "ATA.h"
#include "DEV9/DEV9.h"
@ -24,6 +25,19 @@
#include "HddCreateWx.h"
#endif
#if _WIN32
#include "pathcch.h"
#include <io.h>
#elif defined(__POSIX__)
#define INVALID_HANDLE_VALUE -1
#if defined(__APPLE__)
#include <unistd.h>
#endif
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#endif
ATA::ATA()
{
//Power on, Would do self-Diag + Hardware Init
@ -71,6 +85,8 @@ int ATA::Open(const std::string& hddPath)
//Store HddImage size for later check
hddImageSize = static_cast<u64>(size);
InitSparseSupport(hddPath);
{
std::lock_guard ioSignallock(ioMutex);
ioRead = false;
@ -83,6 +99,139 @@ int ATA::Open(const std::string& hddPath)
return 0;
}
void ATA::InitSparseSupport(const std::string& hddPath)
{
#ifdef _WIN32
hddSparse = false;
const std::wstring wHddPath(StringUtil::UTF8StringToWideString(hddPath));
const DWORD fileAttributes = GetFileAttributes(wHddPath.c_str());
hddSparse = fileAttributes & FILE_ATTRIBUTE_SPARSE_FILE;
if (!hddSparse)
return;
// Get OS specific file handle for spare writing.
// HANDLE is owned by FILE* hddImage.
hddNativeHandle = reinterpret_cast<HANDLE>(_get_osfhandle(_fileno(hddImage)));
if (hddNativeHandle == INVALID_HANDLE_VALUE)
{
Console.Error("DEV9: ATA: Failed to open file for sparse");
hddSparse = false;
return;
}
// Get sparse block size (Initially assumed as 4096 bytes).
hddSparseBlockSize = 4096;
// We need the drive letter for the drive the file actually resides on
// which means we need to deal with any junction links in the path.
DWORD len = GetFinalPathNameByHandle(hddNativeHandle, nullptr, 0, FILE_NAME_NORMALIZED);
if (len != 0)
{
std::unique_ptr<TCHAR[]> name = std::make_unique<TCHAR[]>(len);
len = GetFinalPathNameByHandle(hddNativeHandle, name.get(), len, FILE_NAME_NORMALIZED);
if (len != 0)
{
PCWSTR rootEnd;
if (PathCchSkipRoot(name.get(), &rootEnd) == S_OK)
{
const size_t rootLength = rootEnd - name.get();
std::wstring finalPath(name.get(), rootLength);
DWORD sectorsPerCluster;
DWORD bytesPerSector;
DWORD temp1, temp2;
if (GetDiskFreeSpace(finalPath.c_str(), &sectorsPerCluster, &bytesPerSector, &temp1, &temp2) == TRUE)
hddSparseBlockSize = sectorsPerCluster * bytesPerSector;
else
Console.Error("DEV9: ATA: Failed to get sparse block size (GetDiskFreeSpace() returned false)");
}
else
Console.Error("DEV9: ATA: Failed to get sparse block size (PathCchSkipRoot() returned false)");
}
else
Console.Error("DEV9: ATA: Failed to get sparse block size (PathBuildRoot() returned 0)");
}
else
Console.Error("DEV9: ATA: Failed to get sparse block size (GetFinalPathNameByHandle() returned 0)");
/* https://askbob.tech/the-ntfs-blog-sparse-and-compressed-file/
* NTFS Sparse Block Size are the same size as a compression unit
* Cluster Size Compression Unit
* --------------------------------
* 512bytes 8kb (0x02000)
* 1kb 16kb (0x04000)
* 2kb 32kb (0x08000)
* 4kb 64kb (0x10000)
* 8kb 64kb (0x10000)
* 16kb 64kb (0x10000)
* 32kb 64kb (0x10000)
* 64kb 64kb (0x10000)
* --------------------------------
*/
// Get the filesystem type.
WCHAR fsName[MAX_PATH + 1];
const BOOL ret = GetVolumeInformationByHandleW(hddNativeHandle, nullptr, 0, nullptr, nullptr, nullptr, fsName, MAX_PATH);
if (ret == FALSE)
{
Console.Error("DEV9: ATA: Failed to get sparse block size (GetVolumeInformationByHandle() returned false)");
// Assume NTFS.
wcscpy(fsName, L"NTFS");
}
if ((wcscmp(fsName, L"NTFS") == 0))
{
switch (hddSparseBlockSize)
{
case 512:
hddSparseBlockSize = 8192;
break;
case 1024:
hddSparseBlockSize = 16384;
break;
case 2048:
hddSparseBlockSize = 32768;
break;
case 4096:
case 8192:
case 16384:
case 32768:
case 65536:
hddSparseBlockSize = 65536;
break;
default:
break;
}
}
// Otherwise assume SparseBlockSize == block size.
#elif defined(__POSIX__)
// fd is owned by FILE* hddImage.
hddNativeHandle = fileno(hddImage);
hddSparse = false;
if (hddNativeHandle != -1)
{
// No way to check if we can hole punch without trying it
// so just assume sparse files are supported.
hddSparse = true;
// Get sparse block size (Initially assumed as 4096 bytes).
hddSparseBlockSize = 4096;
struct stat fileInfo;
if (fstat(hddNativeHandle, &fileInfo) == 0)
hddSparseBlockSize = fileInfo.st_blksize;
else
Console.Error("DEV9: ATA: Failed to get sparse block size (fstat returned != 0)");
}
else
Console.Error("DEV9: ATA: Failed to open file for sparse");
#endif
hddSparseBlock = std::make_unique<u8[]>(hddSparseBlockSize);
hddSparseBlockValid = false;
}
void ATA::Close()
{
//Wait for async code to finish
@ -108,6 +257,16 @@ void ATA::Close()
}
//Close File Handle
if (hddSparse)
{
// hddNativeHandle is owned by hddImage.
// It will get closed in fclose(hddImage).
hddNativeHandle = INVALID_HANDLE_VALUE;
hddSparse = false;
hddSparseBlock = nullptr;
hddSparseBlockValid = false;
}
if (hddImage)
{
std::fclose(hddImage);

226
pcsx2/DEV9/ATA/ATA_Transfer.cpp
View File

@ -21,6 +21,12 @@
#include "ATA.h"
#include "DEV9/DEV9.h"
#if __POSIX__
#define INVALID_HANDLE_VALUE -1
#include <unistd.h>
#include <fcntl.h>
#endif
void ATA::IO_Thread()
{
std::unique_lock ioWaitHandle(ioMutex);
@ -99,18 +105,230 @@ bool ATA::IO_Write()
return false;
}
if (FileSystem::FSeek64(hddImage, entry.sector * 512, SEEK_SET) != 0 ||
std::fwrite(entry.data, entry.length, 1, hddImage) != 1 ||
std::fflush(hddImage) != 0)
u64 imagePos = entry.sector * 512;
if (FileSystem::FSeek64(hddImage, imagePos, SEEK_SET) != 0)
{
Console.Error("DEV9: ATA: File write error");
Console.Error("DEV9: ATA: File seek error");
pxAssert(false);
abort();
}
if (hddSparse)
{
u32 written = 0;
while (written != entry.length)
{
IO_SparseCacheUpdateLocation(imagePos + written);
// Align to sparse block size.
u32 writeSize = hddSparseBlockSize - ((imagePos + written) % hddSparseBlockSize);
// Limit to size of write.
writeSize = std::min(writeSize, entry.length - written);
pxAssert(writeSize > 0);
pxAssert(writeSize <= hddSparseBlockSize);
pxAssert((imagePos + written) >= HddSparseStart);
pxAssert((imagePos + written) - HddSparseStart + writeSize <= hddSparseBlockSize);
bool sparseWrite = IsAllZero(&entry.data[written], writeSize);
if (sparseWrite)
{
#if defined(PCSX2_DEBUG) || defined(PCSX2_DEVBUILD)
std::unique_ptr<u8[]> zeroBlock = std::make_unique<u8[]>(writeSize);
memset(zeroBlock.get(), 0, writeSize);
pxAssert(memcmp(&entry.data[written], zeroBlock.get(), writeSize) == 0);
#endif
if (!IO_SparseZero(imagePos + written, writeSize))
{
Console.Error("DEV9: ATA: File sparse write error");
// hddNativeHandle is owned by hddImage.
// do not close it.
hddNativeHandle = INVALID_HANDLE_VALUE;
hddSparse = false;
hddSparseBlock = nullptr;
hddSparseBlockValid = false;
// Fallthough into other if statment.
sparseWrite = false;
}
}
// Also handles sparse write failures.
if (!sparseWrite)
{
#if defined(PCSX2_DEBUG) || defined(PCSX2_DEVBUILD)
if (hddSparse)
{
std::unique_ptr<u8[]> zeroBlock = std::make_unique<u8[]>(writeSize);
memset(zeroBlock.get(), 0, writeSize);
pxAssert(memcmp(&entry.data[written], zeroBlock.get(), writeSize) != 0);
}
#endif
// Update cache.
if (hddSparseBlockValid)
memcpy(&hddSparseBlock[(imagePos + written) - HddSparseStart], &entry.data[written], writeSize);
if (std::fwrite(&entry.data[written], writeSize, 1, hddImage) != 1 ||
std::fflush(hddImage) != 0)
{
Console.Error("DEV9: ATA: File write error");
pxAssert(false);
abort();
}
}
written += writeSize;
pxAssert(FileSystem::FTell64(hddImage) == (s64)(imagePos + written));
}
}
else
{
if (std::fwrite(entry.data, entry.length, 1, hddImage) != 1 || std::fflush(hddImage) != 0)
{
Console.Error("DEV9: ATA: File write error");
pxAssert(false);
abort();
}
}
delete[] entry.data;
return true;
}
void ATA::IO_SparseCacheLoad()
{
// Reads are bounds checked, but for the sectors read only.
// Need to bounds check for sparse block, to handle an edge case of a user providing a file with a size that dosn't align with the sparse block size.
// Normally that won't happen as we generate files of exact Gib size.
u64 readSize = hddSparseBlockSize;
const u64 posEnd = HddSparseStart + hddSparseBlockSize;
if (posEnd > hddImageSize)
{
readSize = hddSparseBlockSize - (posEnd - hddImageSize);
// Zero cache for data beyond end of file.
memset(&hddSparseBlock[readSize], 0, hddSparseBlockSize - readSize);
}
// Store file pointer.
const s64 orgPos = FileSystem::FTell64(hddImage);
// Load into cache.
if (orgPos == -1 ||
FileSystem::FSeek64(hddImage, HddSparseStart, SEEK_SET) != 0 ||
std::fread((char*)hddSparseBlock.get(), readSize, 1, hddImage) != 1 ||
FileSystem::FSeek64(hddImage, orgPos, SEEK_SET) != 0) // Restore file pointer.
{
Console.Error("DEV9: ATA: File read error");
pxAssert(false);
abort();
}
hddSparseBlockValid = true;
}
void ATA::IO_SparseCacheUpdateLocation(u64 byteOffset)
{
const u64 currentBlockStart = (byteOffset / hddSparseBlockSize) * hddSparseBlockSize;
if (currentBlockStart != HddSparseStart)
{
HddSparseStart = currentBlockStart;
hddSparseBlockValid = false;
// Only update cache when we perform a sparse write.
}
}
// Also sets hddImage write ptr.
bool ATA::IO_SparseZero(u64 byteOffset, u64 byteSize)
{
if (hddSparseBlockValid == false)
IO_SparseCacheLoad();
//Assert as range check
pxAssert(byteOffset >= HddSparseStart);
pxAssert(byteOffset - HddSparseStart + byteSize <= hddSparseBlockSize);
//Write to cache
memset(&hddSparseBlock[byteOffset - HddSparseStart], 0, byteSize);
//Is block non-zero?
if (!IsAllZero(hddSparseBlock.get(), hddSparseBlockSize))
{
#if defined(PCSX2_DEBUG) || defined(PCSX2_DEVBUILD)
std::unique_ptr<u8[]> zeroBlock = std::make_unique<u8[]>(hddSparseBlockSize);
memset(zeroBlock.get(), 0, hddSparseBlockSize);
pxAssert(memcmp(hddSparseBlock.get(), zeroBlock.get(), hddSparseBlockSize) != 0);
#endif
//No, do normal write
if (std::fwrite((char*)&hddSparseBlock[byteOffset - HddSparseStart], byteSize, 1, hddImage) != 1 ||
std::fflush(hddImage) != 0)
{
Console.Error("DEV9: ATA: File write error");
pxAssert(false);
abort();
}
return true;
}
#if defined(PCSX2_DEBUG) || defined(PCSX2_DEVBUILD)
std::unique_ptr<u8[]> zeroBlock = std::make_unique<u8[]>(hddSparseBlockSize);
memset(zeroBlock.get(), 0, hddSparseBlockSize);
pxAssert(memcmp(hddSparseBlock.get(), zeroBlock.get(), hddSparseBlockSize) == 0);
#endif
//Yes, try sparse write
#ifdef _WIN32
FILE_ZERO_DATA_INFORMATION sparseRange;
sparseRange.FileOffset.QuadPart = HddSparseStart;
sparseRange.BeyondFinalZero.QuadPart = HddSparseStart + hddSparseBlockSize;
DWORD dwTemp;
const BOOL ret = DeviceIoControl(hddNativeHandle, FSCTL_SET_ZERO_DATA, &sparseRange, sizeof(sparseRange), nullptr, 0, &dwTemp, nullptr);
if (ret == FALSE)
return false;
#elif defined(__linux__)
const int ret = fallocate(hddNativeHandle, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, HddSparseStart, hddSparseBlockSize);
if (ret == -1)
return false;
#elif defined(__APPLE__)
fpunchhole_t sparseRange{0};
sparseRange.fp_offset = HddSparseStart;
sparseRange.fp_length = hddSparseBlockSize;
const int ret = fcntl(hddNativeHandle, F_PUNCHHOLE, &sparseRange);
if (ret == -1)
return false;
#else
Console.Error("DEV9: ATA: Hole punching not supported on current OS");
return false;
#endif
if (FileSystem::FSeek64(hddImage, byteOffset + byteSize, SEEK_SET) != 0)
{
Console.Error("DEV9: ATA: File seek error");
pxAssert(false);
abort();
}
return true;
}
bool ATA::IsAllZero(const void* data, size_t len)
{
intmax_t* pbi = (intmax_t*)data;
intmax_t* pbiUpper = ((intmax_t*)(((char*)data) + len)) - 1;
for (; pbi <= pbiUpper; pbi++)
if (*pbi)
return false; // Check with the biggest int available most of the array, but without aligning it.
for (char* p = (char*)pbi; p < ((char*)data) + len; p++)
if (*p)
return false; // Check end of non aligned array.
return true;
}
void ATA::HDD_ReadAsync(void (ATA::*drqCMD)())
{
nsectorLeft = 0;