mirror of https://github.com/PCSX2/pcsx2.git
DEV9: Allow sparse writing to HDD file
This commit is contained in:
TheLastRar
refractionpcsx2
parent
887a1685dd
commit
31c045fdb5
|
|
@ -21,7 +21,9 @@
|
|||
#include <mutex>
|
||||
#include <condition_variable>
|
||||
|
||||
#include "common/RedtapeWindows.h"
|
||||
#include "common/Path.h"
|
||||
|
||||
#include "DEV9/SimpleQueue.h"
|
||||
|
||||
class ATA
|
||||
|
|
@ -37,6 +39,18 @@ private:
|
|||
std::FILE* hddImage = nullptr;
|
||||
u64 hddImageSize;
|
||||
|
||||
bool hddSparse = false;
|
||||
u64 hddSparseBlockSize;
|
||||
u64 HddSparseStart;
|
||||
std::unique_ptr<u8[]> hddSparseBlock;
|
||||
bool hddSparseBlockValid = false;
|
||||
|
||||
#ifdef _WIN32
|
||||
HANDLE hddNativeHandle = INVALID_HANDLE_VALUE;
|
||||
#elif defined(__POSIX__)
|
||||
int hddNativeHandle = -1;
|
||||
#endif
|
||||
|
||||
int pioMode;
|
||||
int sdmaMode;
|
||||
int mdmaMode;
|
||||
|
|
@ -171,6 +185,8 @@ public:
|
|||
//ATAwritePIO;
|
||||
|
||||
private:
|
||||
void InitSparseSupport(const std::string& hddPath);
|
||||
|
||||
//Info
|
||||
void CreateHDDinfo(u64 sizeSectors);
|
||||
void CreateHDDinfoCsum();
|
||||
|
|
@ -203,6 +219,10 @@ private:
|
|||
void IO_Thread();
|
||||
void IO_Read();
|
||||
bool IO_Write();
|
||||
bool IO_SparseZero(u64 byteOffset, u64 byteSize);
|
||||
void IO_SparseCacheUpdateLocation(u64 Offset);
|
||||
void IO_SparseCacheLoad();
|
||||
bool IsAllZero(const void* data, size_t len);
|
||||
void HDD_ReadAsync(void (ATA::*drqCMD)());
|
||||
void HDD_ReadSync(void (ATA::*drqCMD)());
|
||||
bool HDD_CanAssessOrSetError();
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@
|
|||
|
||||
#include "common/Assertions.h"
|
||||
#include "common/FileSystem.h"
|
||||
#include "common/StringUtil.h"
|
||||
|
||||
#include "ATA.h"
|
||||
#include "DEV9/DEV9.h"
|
||||
|
|
@ -24,6 +25,19 @@
|
|||
#include "HddCreateWx.h"
|
||||
#endif
|
||||
|
||||
#if _WIN32
|
||||
#include "pathcch.h"
|
||||
#include <io.h>
|
||||
#elif defined(__POSIX__)
|
||||
#define INVALID_HANDLE_VALUE -1
|
||||
#if defined(__APPLE__)
|
||||
#include <unistd.h>
|
||||
#endif
|
||||
#include <sys/types.h>
|
||||
#include <sys/stat.h>
|
||||
#include <fcntl.h>
|
||||
#endif
|
||||
|
||||
ATA::ATA()
|
||||
{
|
||||
//Power on, Would do self-Diag + Hardware Init
|
||||
|
|
@ -71,6 +85,8 @@ int ATA::Open(const std::string& hddPath)
|
|||
//Store HddImage size for later check
|
||||
hddImageSize = static_cast<u64>(size);
|
||||
|
||||
InitSparseSupport(hddPath);
|
||||
|
||||
{
|
||||
std::lock_guard ioSignallock(ioMutex);
|
||||
ioRead = false;
|
||||
|
|
@ -83,6 +99,139 @@ int ATA::Open(const std::string& hddPath)
|
|||
return 0;
|
||||
}
|
||||
|
||||
void ATA::InitSparseSupport(const std::string& hddPath)
|
||||
{
|
||||
#ifdef _WIN32
|
||||
hddSparse = false;
|
||||
|
||||
const std::wstring wHddPath(StringUtil::UTF8StringToWideString(hddPath));
|
||||
const DWORD fileAttributes = GetFileAttributes(wHddPath.c_str());
|
||||
hddSparse = fileAttributes & FILE_ATTRIBUTE_SPARSE_FILE;
|
||||
|
||||
if (!hddSparse)
|
||||
return;
|
||||
|
||||
// Get OS specific file handle for spare writing.
|
||||
// HANDLE is owned by FILE* hddImage.
|
||||
hddNativeHandle = reinterpret_cast<HANDLE>(_get_osfhandle(_fileno(hddImage)));
|
||||
if (hddNativeHandle == INVALID_HANDLE_VALUE)
|
||||
{
|
||||
Console.Error("DEV9: ATA: Failed to open file for sparse");
|
||||
hddSparse = false;
|
||||
return;
|
||||
}
|
||||
|
||||
// Get sparse block size (Initially assumed as 4096 bytes).
|
||||
hddSparseBlockSize = 4096;
|
||||
|
||||
// We need the drive letter for the drive the file actually resides on
|
||||
// which means we need to deal with any junction links in the path.
|
||||
DWORD len = GetFinalPathNameByHandle(hddNativeHandle, nullptr, 0, FILE_NAME_NORMALIZED);
|
||||
|
||||
if (len != 0)
|
||||
{
|
||||
std::unique_ptr<TCHAR[]> name = std::make_unique<TCHAR[]>(len);
|
||||
len = GetFinalPathNameByHandle(hddNativeHandle, name.get(), len, FILE_NAME_NORMALIZED);
|
||||
if (len != 0)
|
||||
{
|
||||
PCWSTR rootEnd;
|
||||
if (PathCchSkipRoot(name.get(), &rootEnd) == S_OK)
|
||||
{
|
||||
const size_t rootLength = rootEnd - name.get();
|
||||
std::wstring finalPath(name.get(), rootLength);
|
||||
|
||||
DWORD sectorsPerCluster;
|
||||
DWORD bytesPerSector;
|
||||
DWORD temp1, temp2;
|
||||
if (GetDiskFreeSpace(finalPath.c_str(), §orsPerCluster, &bytesPerSector, &temp1, &temp2) == TRUE)
|
||||
hddSparseBlockSize = sectorsPerCluster * bytesPerSector;
|
||||
else
|
||||
Console.Error("DEV9: ATA: Failed to get sparse block size (GetDiskFreeSpace() returned false)");
|
||||
}
|
||||
else
|
||||
Console.Error("DEV9: ATA: Failed to get sparse block size (PathCchSkipRoot() returned false)");
|
||||
}
|
||||
else
|
||||
Console.Error("DEV9: ATA: Failed to get sparse block size (PathBuildRoot() returned 0)");
|
||||
}
|
||||
else
|
||||
Console.Error("DEV9: ATA: Failed to get sparse block size (GetFinalPathNameByHandle() returned 0)");
|
||||
|
||||
/* https://askbob.tech/the-ntfs-blog-sparse-and-compressed-file/
|
||||
* NTFS Sparse Block Size are the same size as a compression unit
|
||||
* Cluster Size Compression Unit
|
||||
* --------------------------------
|
||||
* 512bytes 8kb (0x02000)
|
||||
* 1kb 16kb (0x04000)
|
||||
* 2kb 32kb (0x08000)
|
||||
* 4kb 64kb (0x10000)
|
||||
* 8kb 64kb (0x10000)
|
||||
* 16kb 64kb (0x10000)
|
||||
* 32kb 64kb (0x10000)
|
||||
* 64kb 64kb (0x10000)
|
||||
* --------------------------------
|
||||
*/
|
||||
|
||||
// Get the filesystem type.
|
||||
WCHAR fsName[MAX_PATH + 1];
|
||||
const BOOL ret = GetVolumeInformationByHandleW(hddNativeHandle, nullptr, 0, nullptr, nullptr, nullptr, fsName, MAX_PATH);
|
||||
if (ret == FALSE)
|
||||
{
|
||||
Console.Error("DEV9: ATA: Failed to get sparse block size (GetVolumeInformationByHandle() returned false)");
|
||||
// Assume NTFS.
|
||||
wcscpy(fsName, L"NTFS");
|
||||
}
|
||||
if ((wcscmp(fsName, L"NTFS") == 0))
|
||||
{
|
||||
switch (hddSparseBlockSize)
|
||||
{
|
||||
case 512:
|
||||
hddSparseBlockSize = 8192;
|
||||
break;
|
||||
case 1024:
|
||||
hddSparseBlockSize = 16384;
|
||||
break;
|
||||
case 2048:
|
||||
hddSparseBlockSize = 32768;
|
||||
break;
|
||||
case 4096:
|
||||
case 8192:
|
||||
case 16384:
|
||||
case 32768:
|
||||
case 65536:
|
||||
hddSparseBlockSize = 65536;
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
// Otherwise assume SparseBlockSize == block size.
|
||||
|
||||
#elif defined(__POSIX__)
|
||||
// fd is owned by FILE* hddImage.
|
||||
hddNativeHandle = fileno(hddImage);
|
||||
hddSparse = false;
|
||||
if (hddNativeHandle != -1)
|
||||
{
|
||||
// No way to check if we can hole punch without trying it
|
||||
// so just assume sparse files are supported.
|
||||
hddSparse = true;
|
||||
|
||||
// Get sparse block size (Initially assumed as 4096 bytes).
|
||||
hddSparseBlockSize = 4096;
|
||||
struct stat fileInfo;
|
||||
if (fstat(hddNativeHandle, &fileInfo) == 0)
|
||||
hddSparseBlockSize = fileInfo.st_blksize;
|
||||
else
|
||||
Console.Error("DEV9: ATA: Failed to get sparse block size (fstat returned != 0)");
|
||||
}
|
||||
else
|
||||
Console.Error("DEV9: ATA: Failed to open file for sparse");
|
||||
#endif
|
||||
hddSparseBlock = std::make_unique<u8[]>(hddSparseBlockSize);
|
||||
hddSparseBlockValid = false;
|
||||
}
|
||||
|
||||
void ATA::Close()
|
||||
{
|
||||
//Wait for async code to finish
|
||||
|
|
@ -108,6 +257,16 @@ void ATA::Close()
|
|||
}
|
||||
|
||||
//Close File Handle
|
||||
if (hddSparse)
|
||||
{
|
||||
// hddNativeHandle is owned by hddImage.
|
||||
// It will get closed in fclose(hddImage).
|
||||
hddNativeHandle = INVALID_HANDLE_VALUE;
|
||||
|
||||
hddSparse = false;
|
||||
hddSparseBlock = nullptr;
|
||||
hddSparseBlockValid = false;
|
||||
}
|
||||
if (hddImage)
|
||||
{
|
||||
std::fclose(hddImage);
|
||||
|
|
|
|||
|
|
@ -21,6 +21,12 @@
|
|||
#include "ATA.h"
|
||||
#include "DEV9/DEV9.h"
|
||||
|
||||
#if __POSIX__
|
||||
#define INVALID_HANDLE_VALUE -1
|
||||
#include <unistd.h>
|
||||
#include <fcntl.h>
|
||||
#endif
|
||||
|
||||
void ATA::IO_Thread()
|
||||
{
|
||||
std::unique_lock ioWaitHandle(ioMutex);
|
||||
|
|
@ -99,18 +105,230 @@ bool ATA::IO_Write()
|
|||
return false;
|
||||
}
|
||||
|
||||
if (FileSystem::FSeek64(hddImage, entry.sector * 512, SEEK_SET) != 0 ||
|
||||
std::fwrite(entry.data, entry.length, 1, hddImage) != 1 ||
|
||||
u64 imagePos = entry.sector * 512;
|
||||
if (FileSystem::FSeek64(hddImage, imagePos, SEEK_SET) != 0)
|
||||
{
|
||||
Console.Error("DEV9: ATA: File seek error");
|
||||
pxAssert(false);
|
||||
abort();
|
||||
}
|
||||
if (hddSparse)
|
||||
{
|
||||
u32 written = 0;
|
||||
while (written != entry.length)
|
||||
{
|
||||
IO_SparseCacheUpdateLocation(imagePos + written);
|
||||
// Align to sparse block size.
|
||||
u32 writeSize = hddSparseBlockSize - ((imagePos + written) % hddSparseBlockSize);
|
||||
// Limit to size of write.
|
||||
writeSize = std::min(writeSize, entry.length - written);
|
||||
|
||||
pxAssert(writeSize > 0);
|
||||
pxAssert(writeSize <= hddSparseBlockSize);
|
||||
pxAssert((imagePos + written) >= HddSparseStart);
|
||||
pxAssert((imagePos + written) - HddSparseStart + writeSize <= hddSparseBlockSize);
|
||||
|
||||
bool sparseWrite = IsAllZero(&entry.data[written], writeSize);
|
||||
|
||||
if (sparseWrite)
|
||||
{
|
||||
#if defined(PCSX2_DEBUG) || defined(PCSX2_DEVBUILD)
|
||||
std::unique_ptr<u8[]> zeroBlock = std::make_unique<u8[]>(writeSize);
|
||||
memset(zeroBlock.get(), 0, writeSize);
|
||||
pxAssert(memcmp(&entry.data[written], zeroBlock.get(), writeSize) == 0);
|
||||
#endif
|
||||
|
||||
if (!IO_SparseZero(imagePos + written, writeSize))
|
||||
{
|
||||
Console.Error("DEV9: ATA: File sparse write error");
|
||||
|
||||
// hddNativeHandle is owned by hddImage.
|
||||
// do not close it.
|
||||
hddNativeHandle = INVALID_HANDLE_VALUE;
|
||||
|
||||
hddSparse = false;
|
||||
hddSparseBlock = nullptr;
|
||||
hddSparseBlockValid = false;
|
||||
|
||||
// Fallthough into other if statment.
|
||||
sparseWrite = false;
|
||||
}
|
||||
}
|
||||
|
||||
// Also handles sparse write failures.
|
||||
if (!sparseWrite)
|
||||
{
|
||||
#if defined(PCSX2_DEBUG) || defined(PCSX2_DEVBUILD)
|
||||
if (hddSparse)
|
||||
{
|
||||
std::unique_ptr<u8[]> zeroBlock = std::make_unique<u8[]>(writeSize);
|
||||
memset(zeroBlock.get(), 0, writeSize);
|
||||
pxAssert(memcmp(&entry.data[written], zeroBlock.get(), writeSize) != 0);
|
||||
}
|
||||
#endif
|
||||
// Update cache.
|
||||
if (hddSparseBlockValid)
|
||||
memcpy(&hddSparseBlock[(imagePos + written) - HddSparseStart], &entry.data[written], writeSize);
|
||||
|
||||
if (std::fwrite(&entry.data[written], writeSize, 1, hddImage) != 1 ||
|
||||
std::fflush(hddImage) != 0)
|
||||
{
|
||||
Console.Error("DEV9: ATA: File write error");
|
||||
pxAssert(false);
|
||||
abort();
|
||||
}
|
||||
}
|
||||
written += writeSize;
|
||||
pxAssert(FileSystem::FTell64(hddImage) == (s64)(imagePos + written));
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
if (std::fwrite(entry.data, entry.length, 1, hddImage) != 1 || std::fflush(hddImage) != 0)
|
||||
{
|
||||
Console.Error("DEV9: ATA: File write error");
|
||||
pxAssert(false);
|
||||
abort();
|
||||
}
|
||||
}
|
||||
delete[] entry.data;
|
||||
return true;
|
||||
}
|
||||
|
||||
void ATA::IO_SparseCacheLoad()
|
||||
{
|
||||
// Reads are bounds checked, but for the sectors read only.
|
||||
// Need to bounds check for sparse block, to handle an edge case of a user providing a file with a size that dosn't align with the sparse block size.
|
||||
// Normally that won't happen as we generate files of exact Gib size.
|
||||
u64 readSize = hddSparseBlockSize;
|
||||
const u64 posEnd = HddSparseStart + hddSparseBlockSize;
|
||||
if (posEnd > hddImageSize)
|
||||
{
|
||||
readSize = hddSparseBlockSize - (posEnd - hddImageSize);
|
||||
// Zero cache for data beyond end of file.
|
||||
memset(&hddSparseBlock[readSize], 0, hddSparseBlockSize - readSize);
|
||||
}
|
||||
|
||||
// Store file pointer.
|
||||
const s64 orgPos = FileSystem::FTell64(hddImage);
|
||||
|
||||
// Load into cache.
|
||||
if (orgPos == -1 ||
|
||||
FileSystem::FSeek64(hddImage, HddSparseStart, SEEK_SET) != 0 ||
|
||||
std::fread((char*)hddSparseBlock.get(), readSize, 1, hddImage) != 1 ||
|
||||
FileSystem::FSeek64(hddImage, orgPos, SEEK_SET) != 0) // Restore file pointer.
|
||||
{
|
||||
Console.Error("DEV9: ATA: File read error");
|
||||
pxAssert(false);
|
||||
abort();
|
||||
}
|
||||
|
||||
hddSparseBlockValid = true;
|
||||
}
|
||||
|
||||
void ATA::IO_SparseCacheUpdateLocation(u64 byteOffset)
|
||||
{
|
||||
const u64 currentBlockStart = (byteOffset / hddSparseBlockSize) * hddSparseBlockSize;
|
||||
if (currentBlockStart != HddSparseStart)
|
||||
{
|
||||
HddSparseStart = currentBlockStart;
|
||||
hddSparseBlockValid = false;
|
||||
// Only update cache when we perform a sparse write.
|
||||
}
|
||||
}
|
||||
|
||||
// Also sets hddImage write ptr.
|
||||
bool ATA::IO_SparseZero(u64 byteOffset, u64 byteSize)
|
||||
{
|
||||
if (hddSparseBlockValid == false)
|
||||
IO_SparseCacheLoad();
|
||||
|
||||
//Assert as range check
|
||||
pxAssert(byteOffset >= HddSparseStart);
|
||||
pxAssert(byteOffset - HddSparseStart + byteSize <= hddSparseBlockSize);
|
||||
|
||||
//Write to cache
|
||||
memset(&hddSparseBlock[byteOffset - HddSparseStart], 0, byteSize);
|
||||
|
||||
//Is block non-zero?
|
||||
if (!IsAllZero(hddSparseBlock.get(), hddSparseBlockSize))
|
||||
{
|
||||
#if defined(PCSX2_DEBUG) || defined(PCSX2_DEVBUILD)
|
||||
std::unique_ptr<u8[]> zeroBlock = std::make_unique<u8[]>(hddSparseBlockSize);
|
||||
memset(zeroBlock.get(), 0, hddSparseBlockSize);
|
||||
pxAssert(memcmp(hddSparseBlock.get(), zeroBlock.get(), hddSparseBlockSize) != 0);
|
||||
#endif
|
||||
|
||||
//No, do normal write
|
||||
if (std::fwrite((char*)&hddSparseBlock[byteOffset - HddSparseStart], byteSize, 1, hddImage) != 1 ||
|
||||
std::fflush(hddImage) != 0)
|
||||
{
|
||||
Console.Error("DEV9: ATA: File write error");
|
||||
pxAssert(false);
|
||||
abort();
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
#if defined(PCSX2_DEBUG) || defined(PCSX2_DEVBUILD)
|
||||
std::unique_ptr<u8[]> zeroBlock = std::make_unique<u8[]>(hddSparseBlockSize);
|
||||
memset(zeroBlock.get(), 0, hddSparseBlockSize);
|
||||
pxAssert(memcmp(hddSparseBlock.get(), zeroBlock.get(), hddSparseBlockSize) == 0);
|
||||
#endif
|
||||
|
||||
//Yes, try sparse write
|
||||
#ifdef _WIN32
|
||||
FILE_ZERO_DATA_INFORMATION sparseRange;
|
||||
sparseRange.FileOffset.QuadPart = HddSparseStart;
|
||||
sparseRange.BeyondFinalZero.QuadPart = HddSparseStart + hddSparseBlockSize;
|
||||
DWORD dwTemp;
|
||||
const BOOL ret = DeviceIoControl(hddNativeHandle, FSCTL_SET_ZERO_DATA, &sparseRange, sizeof(sparseRange), nullptr, 0, &dwTemp, nullptr);
|
||||
|
||||
if (ret == FALSE)
|
||||
return false;
|
||||
|
||||
#elif defined(__linux__)
|
||||
const int ret = fallocate(hddNativeHandle, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, HddSparseStart, hddSparseBlockSize);
|
||||
|
||||
if (ret == -1)
|
||||
return false;
|
||||
|
||||
#elif defined(__APPLE__)
|
||||
fpunchhole_t sparseRange{0};
|
||||
sparseRange.fp_offset = HddSparseStart;
|
||||
sparseRange.fp_length = hddSparseBlockSize;
|
||||
|
||||
const int ret = fcntl(hddNativeHandle, F_PUNCHHOLE, &sparseRange);
|
||||
|
||||
if (ret == -1)
|
||||
return false;
|
||||
|
||||
#else
|
||||
Console.Error("DEV9: ATA: Hole punching not supported on current OS");
|
||||
return false;
|
||||
#endif
|
||||
if (FileSystem::FSeek64(hddImage, byteOffset + byteSize, SEEK_SET) != 0)
|
||||
{
|
||||
Console.Error("DEV9: ATA: File seek error");
|
||||
pxAssert(false);
|
||||
abort();
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool ATA::IsAllZero(const void* data, size_t len)
|
||||
{
|
||||
intmax_t* pbi = (intmax_t*)data;
|
||||
intmax_t* pbiUpper = ((intmax_t*)(((char*)data) + len)) - 1;
|
||||
for (; pbi <= pbiUpper; pbi++)
|
||||
if (*pbi)
|
||||
return false; // Check with the biggest int available most of the array, but without aligning it.
|
||||
for (char* p = (char*)pbi; p < ((char*)data) + len; p++)
|
||||
if (*p)
|
||||
return false; // Check end of non aligned array.
|
||||
return true;
|
||||
}
|
||||
|
||||
void ATA::HDD_ReadAsync(void (ATA::*drqCMD)())
|
||||
{
|
||||
nsectorLeft = 0;
|
||||
|
|
|
|||
Loading…
Reference in New Issue