Indexed gzipped ISO support (slow - no caching)

This commit is contained in:
Avi Halachmi (:avih) 2014-04-27 19:58:20 +03:00
parent 7d491cb230
commit 5771e6eae8
7 changed files with 399 additions and 85 deletions

View File

@ -95,6 +95,22 @@ public:
virtual void SetDataOffset(uint bytes) { m_dataoffset = bytes; }
};
// Factory - creates an AsyncFileReader derived instance which can read a compressed file
class CompressedFileReader {
public:
// Checks if any of the available compressed file handlers can open this
static bool DetectCompressed(AsyncFileReader* pReader);
// fileName is only used to choose the compressed reader.
// If no matching handler is found then an arbitrary handlers will be returned.
// The returned instance still needs ->Open(filename) before usage.
// Open(filename) may still fail.
static AsyncFileReader* GetNewReader(const wxString& fileName);
private:
virtual ~CompressedFileReader() = 0;
};
class MultipartFileReader : public AsyncFileReader
{
DeclareNoncopyableObject( MultipartFileReader );

View File

@ -0,0 +1,257 @@
/* PCSX2 - PS2 Emulator for PCs
* Copyright (C) 2002-2014 PCSX2 Dev Team
*
* PCSX2 is free software: you can redistribute it and/or modify it under the terms
* of the GNU Lesser General Public License as published by the Free Software Found-
* ation, either version 3 of the License, or (at your option) any later version.
*
* PCSX2 is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
* without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
* PURPOSE. See the GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License along with PCSX2.
* If not, see <http://www.gnu.org/licenses/>.
*/
#include "PrecompiledHeader.h"
#include "AsyncFileReader.h"
#include "zlib_indexed.c"
/////////// Some complementary utilities for zlib_indexed.c //////////
#include <fstream>
static s64 fsize(const wxString& filename) {
if (!wxFileName::FileExists(filename))
return -1;
std::ifstream f(filename.ToUTF8(), std::ifstream::binary);
f.seekg(0, f.end);
s64 size = f.tellg();
f.close();
return size;
}
#define GZIP_ID "PCSX2.index.gzip.v1|"
#define GZIP_ID_LEN (sizeof(GZIP_ID) - 1) /* sizeof includes the \0 terminator */
// File format is:
// - [GZIP_ID_LEN] GZIP_ID (no \0)
// - [sizeof(Access)] index (should be allocated, contains various sizes)
// - [rest] the indexed data points (should be allocated, index->list should then point to it)
static Access* ReadIndexFromFile(const wxString& filename) {
s64 size = fsize(filename);
if (size <= 0) {
Console.Error("Error: Can't open index file: '%s'", filename.To8BitData());
return 0;
}
std::ifstream infile(filename.ToUTF8(), std::ifstream::binary);
char fileId[GZIP_ID_LEN + 1] = { 0 };
infile.read(fileId, GZIP_ID_LEN);
if (wxString::From8BitData(GZIP_ID) != wxString::From8BitData(fileId)) {
Console.Error("Error: Incompatible gzip index, please delete it manually: '%s'", filename.To8BitData());
infile.close();
return 0;
}
Access* index = (Access*)malloc(sizeof(Access));
infile.read((char*)index, sizeof(Access));
s64 datasize = size - GZIP_ID_LEN - sizeof(Access);
if (datasize != index->have * sizeof(Point)) {
Console.Error("Error: unexpected size of gzip index, please delete it manually: '%s'.", filename.To8BitData());
infile.close();
free(index);
return 0;
}
char* buffer = (char*)malloc(datasize);
infile.read(buffer, datasize);
infile.close();
index->list = (Point*)buffer; // adjust list pointer
return index;
}
static void WriteIndexToFile(Access* index, const wxString filename) {
if (wxFileName::FileExists(filename)) {
Console.Warning("WARNING: Won't write index - file name exists (please delete it manually): '%s'", filename.To8BitData());
return;
}
std::ofstream outfile(filename.ToUTF8(), std::ofstream::binary);
outfile.write(GZIP_ID, GZIP_ID_LEN);
Point* tmp = index->list;
index->list = 0; // current pointer is useless on disk, normalize it as 0.
outfile.write((char*)index, sizeof(Access));
index->list = tmp;
outfile.write((char*)index->list, sizeof(Point) * index->have);
outfile.close();
// Verify
if (fsize(filename) != (s64)GZIP_ID_LEN + sizeof(Access) + sizeof(Point) * index->have) {
Console.Warning("Warning: Can't write index file to disk: '%s'", filename.To8BitData());
} else {
Console.WriteLn(Color_Green, "OK: Gzip quick access index file saved to disk: '%s'", filename.To8BitData());
}
}
/////////// End of complementary utilities for zlib_indexed.c //////////
static wxString iso2indexname(const wxString& isoname) {
return isoname + L".pindex.tmp";
}
static void WarnOldIndex(const wxString& filename) {
wxString oldName = filename + L".pcsx2.index.tmp";
if (wxFileName::FileExists(oldName)) {
Console.Warning("Note: Unused old index detected, please delete it manually: '%s'", oldName.To8BitData());
}
}
class GzippedFileReader : public AsyncFileReader
{
DeclareNoncopyableObject(GzippedFileReader);
public:
GzippedFileReader(void) :
m_pIndex(0)
{ m_blocksize = 2048; };
virtual ~GzippedFileReader(void) { Close(); };
static bool CanHandle(const wxString& fileName);
virtual bool Open(const wxString& fileName);
virtual int ReadSync(void* pBuffer, uint sector, uint count);
virtual void BeginRead(void* pBuffer, uint sector, uint count);
virtual int FinishRead(void);
virtual void CancelRead(void) {};
virtual void Close(void);
virtual uint GetBlockCount(void) const {
// type and formula copied from FlatFileReader
// FIXME? : Shouldn't it be uint and (size - m_dataoffset) / m_blocksize ?
return (int)((m_pIndex ? m_pIndex->uncompressed_size : 0) / m_blocksize);
};
// Same as FlatFileReader, but in case it changes
virtual void SetBlockSize(uint bytes) { m_blocksize = bytes; }
virtual void SetDataOffset(uint bytes) { m_dataoffset = bytes; }
private:
bool OkIndex(); // Verifies thatt we have an index, or try to create one
int mBytesRead; // Temp sync read result when simulating async read
Access* m_pIndex; // Quick access index
};
// TODO: do better than just checking existance and extension
bool GzippedFileReader::CanHandle(const wxString& fileName) {
return wxFileName::FileExists(fileName) && fileName.Lower().EndsWith(L".gz");
}
#define SPAN_DEFAULT (1048576L * 2) /* distance between access points when creating a new index */
bool GzippedFileReader::OkIndex() {
if (m_pIndex)
return true;
// Try to read index from disk
WarnOldIndex(m_filename);
wxString indexfile = iso2indexname(m_filename);
if (wxFileName::FileExists(indexfile) && (m_pIndex = ReadIndexFromFile(indexfile))) {
Console.WriteLn(Color_Green, "OK: Gzip quick access index read from disk: '%s'", indexfile.To8BitData());
if (m_pIndex->span != SPAN_DEFAULT) {
Console.Warning("Note: This index has %1.1f MB intervals, while the current default for new indexes is %1.1f MB.", (float)m_pIndex->span / 1024 / 1024, (float)SPAN_DEFAULT / 1024 / 1024);
Console.Warning("It will work fine, but if you want to generate a new index with default intervals, delete this index file.");
Console.Warning("(smaller intervals mean bigger index file and quicker but more frequent decompressions)");
}
return true;
}
// No valid index file. Generate an index
Console.Warning("This may take a while (but only once). Scanning compressed file to generate a quick access index...");
Access *index;
FILE* infile = fopen(m_filename.ToUTF8(), "rb");
int len = build_index(infile, SPAN_DEFAULT, &index);
printf("\n"); // build_index prints progress without \n's
fclose(infile);
if (len >= 0) {
m_pIndex = index;
WriteIndexToFile((Access*)m_pIndex, indexfile);
} else {
Console.Error("ERROR (%d): index could not be generated for file '%s'", len, m_filename.To8BitData());
return false;
}
return true;
}
bool GzippedFileReader::Open(const wxString& fileName) {
Close();
m_filename = fileName;
if (!CanHandle(fileName) || !OkIndex()) {
Close();
return false;
};
return true;
};
void GzippedFileReader::BeginRead(void* pBuffer, uint sector, uint count) {
// No a-sync support yet, implement as sync
mBytesRead = ReadSync(pBuffer, sector, count);
return;
};
int GzippedFileReader::FinishRead(void) {
int res = mBytesRead;
mBytesRead = -1;
return res;
};
int GzippedFileReader::ReadSync(void* pBuffer, uint sector, uint count) {
if (!OkIndex())
return -1;
PX_off_t offset = (s64)sector * m_blocksize + m_dataoffset;
int bytesToRead = count * m_blocksize;
FILE* in = fopen(m_filename.ToUTF8(), "rb");
int res = extract(in, m_pIndex, offset, (unsigned char*)pBuffer, bytesToRead);
fclose(in);
return res;
}
void GzippedFileReader::Close() {
m_filename.Empty();
if (m_pIndex) {
free_index((Access*)m_pIndex);
m_pIndex = 0;
}
}
// CompressedFileReader factory - currently there's only GzippedFileReader
// Go through available compressed readers
bool CompressedFileReader::DetectCompressed(AsyncFileReader* pReader) {
return GzippedFileReader::CanHandle(pReader->GetFilename());
}
// Return a new reader which can handle, or any reader otherwise (which will fail on open)
AsyncFileReader* CompressedFileReader::GetNewReader(const wxString& fileName) {
//if (GzippedFileReader::CanHandle(pReader))
return new GzippedFileReader();
}

View File

@ -204,7 +204,7 @@ bool InputIsoFile::Open( const wxString& srcfile, bool testOnly )
m_reader = new FlatFileReader();
m_reader->Open(m_filename);
bool isBlockdump;
bool isBlockdump, isCompressed = false;
if(isBlockdump = BlockdumpFileReader::DetectBlockdump(m_reader))
{
delete m_reader;
@ -219,6 +219,10 @@ bool InputIsoFile::Open( const wxString& srcfile, bool testOnly )
m_reader = bdr;
ReadUnit = 1;
} else if (isCompressed = CompressedFileReader::DetectCompressed(m_reader)) {
delete m_reader;
m_reader = CompressedFileReader::GetNewReader(m_filename);
m_reader->Open(m_filename);
}
bool detected = Detect();
@ -231,7 +235,7 @@ bool InputIsoFile::Open( const wxString& srcfile, bool testOnly )
.SetUserMsg(_("Unrecognized ISO image file format"))
.SetDiagMsg(L"ISO mounting failed: PCSX2 is unable to identify the ISO image type.");
if(!isBlockdump)
if(!isBlockdump && !isCompressed)
{
ReadUnit = MaxReadUnit;

View File

@ -1,3 +1,32 @@
/* zran.c -- example of zlib/gzip stream indexing and random access
Copyright (C) 2005, 2012 Mark Adler
This software is provided 'as-is', without any express or implied
warranty. In no event will the authors be held liable for any damages
arising from the use of this software.
Permission is granted to anyone to use this software for any purpose,
including commercial applications, and to alter it and redistribute it
freely, subject to the following restrictions:
1. The origin of this software must not be misrepresented; you must not
claim that you wrote the original software. If you use this software
in a product, an acknowledgment in the product documentation would be
appreciated but is not required.
2. Altered source versions must be plainly marked as such, and must not be
misrepresented as being the original software.
3. This notice may not be removed or altered from any source distribution.
Jean-loup Gailly Mark Adler
jloup@gzip.org madler@alumni.caltech.edu
The data format used by the zlib library is described by RFCs (Request for
Comments) 1950 to 1952 in the files http://tools.ietf.org/html/rfc1950
(zlib format), rfc1951 (deflate format) and rfc1952 (gzip format).
*/
/* zran.c -- example of zlib/gzip stream indexing and random access
* Copyright (C) 2005, 2012 Mark Adler
* For conditions of distribution and use, see copyright notice in zlib.h
@ -6,6 +35,18 @@
/* Version History:
1.0 29 May 2005 First version
1.1 29 Sep 2012 Fix memory reallocation error
1.1+ 16 Apr 2014 PCSX2 adaptation
(This is verbatim copy from zlib/examples/zran.c, with the following mods):
- Added an explicit license clause taken from zlib.h and removed the sample main(...).
- zlib include path and included Pcsx2Types.h for windows off_t (s64)
- fseeko and off_t #define'ed for windows too (on windows off_t is 32b and no fseeko)
- typedefs for struct access/point (Access/Point) and allocation type casts
- access: added members span and uncompressed_size which are filled by build_index.
- point and access packed for safety since they go to disk as is (but no endian-ness handling).
But they're still aligned since each member size is multiple of 4, so no perf issues.
- build_index(...) - added progress prints
- CHUNK changed from 16k to 512k
*/
/* Illustrate the use of Z_BLOCK, inflatePrime(), and inflateSetDictionary()
@ -52,31 +93,66 @@
index in a file.
*/
#ifndef __ZLIB_INDEXED_C__
#define __ZLIB_INDEXED_C__
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "zlib.h"
#include <zlib/zlib.h>
#include <Pcsx2Types.h>
#ifdef WIN32
# define PX_fseeko _fseeki64
# define PX_off_t s64 /* __int64 */
#else
# define PX_fseeko fseeko
# define PX_off_t off_t
#endif
#define local static
#define SPAN 1048576L /* desired distance between access points */
//#define SPAN (1048576L*2) /* desired distance between access points */
#define WINSIZE 32768U /* sliding window size */
#define CHUNK 16384 /* file input buffer size */
#define CHUNK (512 * 1024) /* file input buffer size */
#ifdef WIN32
# pragma pack(push, indexData, 1)
#endif
/* access point entry */
struct point {
off_t out; /* corresponding offset in uncompressed data */
off_t in; /* offset in input file of first full byte */
PX_off_t out; /* corresponding offset in uncompressed data */
PX_off_t in; /* offset in input file of first full byte */
int bits; /* number of bits (1-7) from byte at in - 1, or 0 */
unsigned char window[WINSIZE]; /* preceding 32K of uncompressed data */
};
}
#ifndef WIN32
__attribute__((packed))
#endif
;
typedef struct point Point;
/* access point list */
struct access {
int have; /* number of list entries filled in */
int size; /* number of list entries allocated */
int size; /* number of list entries allocated (only used internally during build)*/
struct point *list; /* allocated list */
};
s32 span; /* once the index is built, holds the span size used to build it */
PX_off_t uncompressed_size; /* filled by build_index */
}
#ifndef WIN32
__attribute__((packed))
#endif
;
typedef struct access Access;
#ifdef WIN32
# pragma pack(pop, indexData)
#endif
/* Deallocate an index built by build_index() */
local void free_index(struct access *index)
@ -90,15 +166,15 @@ local void free_index(struct access *index)
/* Add an entry to the access point list. If out of memory, deallocate the
existing list and return NULL. */
local struct access *addpoint(struct access *index, int bits,
off_t in, off_t out, unsigned left, unsigned char *window)
PX_off_t in, PX_off_t out, unsigned left, unsigned char *window)
{
struct point *next;
/* if list is empty, create it (start with eight points) */
if (index == NULL) {
index = malloc(sizeof(struct access));
index = (Access*)malloc(sizeof(struct access));
if (index == NULL) return NULL;
index->list = malloc(sizeof(struct point) << 3);
index->list = (Point*)malloc(sizeof(struct point) << 3);
if (index->list == NULL) {
free(index);
return NULL;
@ -110,7 +186,7 @@ local struct access *addpoint(struct access *index, int bits,
/* if list is full, make it bigger */
else if (index->have == index->size) {
index->size <<= 1;
next = realloc(index->list, sizeof(struct point) * index->size);
next = (Point*)realloc(index->list, sizeof(struct point) * index->size);
if (next == NULL) {
free_index(index);
return NULL;
@ -141,11 +217,11 @@ local struct access *addpoint(struct access *index, int bits,
returns the number of access points on success (>= 1), Z_MEM_ERROR for out
of memory, Z_DATA_ERROR for an error in the input file, or Z_ERRNO for a
file read error. On success, *built points to the resulting index. */
local int build_index(FILE *in, off_t span, struct access **built)
local int build_index(FILE *in, PX_off_t span, struct access **built)
{
int ret;
off_t totin, totout; /* our own total counters to avoid 4GB limit */
off_t last; /* totout value of last access point */
PX_off_t totin, totout, totPrinted; /* our own total counters to avoid 4GB limit */
PX_off_t last; /* totout value of last access point */
struct access *index; /* access points being generated */
z_stream strm;
unsigned char input[CHUNK];
@ -164,7 +240,7 @@ local int build_index(FILE *in, off_t span, struct access **built)
/* inflate the input, maintain a sliding window, and build an index -- this
also validates the integrity of the compressed data using the check
information at the end of the gzip or zlib stream */
totin = totout = last = 0;
totin = totout = last = totPrinted = 0;
index = NULL; /* will be allocated by first addpoint() */
strm.avail_out = 0;
do {
@ -222,14 +298,20 @@ local int build_index(FILE *in, off_t span, struct access **built)
last = totout;
}
} while (strm.avail_in != 0);
if (totin / (50 * 1024 * 1024) != totPrinted / (50 * 1024 * 1024)) {
printf("%dMB ", (int)(totin / (1024 * 1024)));
totPrinted = totin;
}
} while (ret != Z_STREAM_END);
/* clean up and return index (release unused entries in list) */
(void)inflateEnd(&strm);
index->list = realloc(index->list, sizeof(struct point) * index->have);
index->list = (Point*)realloc(index->list, sizeof(struct point) * index->have);
index->size = index->have;
index->span = span;
index->uncompressed_size = totout;
*built = index;
return index->size;
return index->have;
/* return error */
build_index_error:
@ -246,7 +328,7 @@ local int build_index(FILE *in, off_t span, struct access **built)
should not return a data error unless the file was modified since the index
was generated. extract() may also return Z_ERRNO if there is an error on
reading or seeking the input file. */
local int extract(FILE *in, struct access *index, off_t offset,
local int extract(FILE *in, struct access *index, PX_off_t offset,
unsigned char *buf, int len)
{
int ret, skip;
@ -274,7 +356,7 @@ local int extract(FILE *in, struct access *index, off_t offset,
ret = inflateInit2(&strm, -15); /* raw inflate */
if (ret != Z_OK)
return ret;
ret = fseeko(in, here->in - (here->bits ? 1 : 0), SEEK_SET);
ret = PX_fseeko(in, here->in - (here->bits ? 1 : 0), SEEK_SET);
if (ret == -1)
goto extract_ret;
if (here->bits) {
@ -348,62 +430,4 @@ local int extract(FILE *in, struct access *index, off_t offset,
return ret;
}
/* Demonstrate the use of build_index() and extract() by processing the file
provided on the command line, and the extracting 16K from about 2/3rds of
the way through the uncompressed output, and writing that to stdout. */
int main(int argc, char **argv)
{
int len;
off_t offset;
FILE *in;
struct access *index = NULL;
unsigned char buf[CHUNK];
/* open input file */
if (argc != 2) {
fprintf(stderr, "usage: zran file.gz\n");
return 1;
}
in = fopen(argv[1], "rb");
if (in == NULL) {
fprintf(stderr, "zran: could not open %s for reading\n", argv[1]);
return 1;
}
/* build index */
len = build_index(in, SPAN, &index);
if (len < 0) {
fclose(in);
switch (len) {
case Z_MEM_ERROR:
fprintf(stderr, "zran: out of memory\n");
break;
case Z_DATA_ERROR:
fprintf(stderr, "zran: compressed data error in %s\n", argv[1]);
break;
case Z_ERRNO:
fprintf(stderr, "zran: read error on %s\n", argv[1]);
break;
default:
fprintf(stderr, "zran: error %d while building index\n", len);
}
return 1;
}
fprintf(stderr, "zran: built index with %d access points\n", len);
/* use index by reading some bytes from an arbitrary offset */
offset = (index->list[index->have - 1].out << 1) / 3;
len = extract(in, index, offset, buf, CHUNK);
if (len < 0)
fprintf(stderr, "zran: extraction failed: %s error\n",
len == Z_MEM_ERROR ? "out of memory" : "input corrupted");
else {
fwrite(buf, 1, len, stdout);
fprintf(stderr, "zran: extracted %d bytes at %llu\n", len, offset);
}
/* clean up and exit */
free_index(index);
fclose(in);
return 0;
}
#endif /* __ZLIB_INDEXED_C__ */

View File

@ -255,8 +255,8 @@ bool MainEmuFrame::_DoSelectIsoBrowser( wxString& result )
wxArrayString isoFilterTypes;
isoFilterTypes.Add(pxsFmt(_("All Supported (%s)"), (isoSupportedLabel + L" .dump").c_str()));
isoFilterTypes.Add(isoSupportedList + L";*.dump");
isoFilterTypes.Add(pxsFmt(_("All Supported (%s)"), (isoSupportedLabel + L" .dump" + L" .gz").c_str()));
isoFilterTypes.Add(isoSupportedList + L";*.dump" + L";*.gz");
isoFilterTypes.Add(pxsFmt(_("Disc Images (%s)"), isoSupportedLabel.c_str() ));
isoFilterTypes.Add(isoSupportedList);
@ -264,6 +264,9 @@ bool MainEmuFrame::_DoSelectIsoBrowser( wxString& result )
isoFilterTypes.Add(pxsFmt(_("Blockdumps (%s)"), L".dump" ));
isoFilterTypes.Add(L"*.dump");
isoFilterTypes.Add(pxsFmt(_("Compressed (%s)"), L".gz"));
isoFilterTypes.Add(L"*.gz");
isoFilterTypes.Add(_("All Files (*.*)"));
isoFilterTypes.Add(L"*.*");

View File

@ -406,7 +406,9 @@
</ItemGroup>
<ItemGroup>
<ClCompile Include="..\..\CDVD\BlockdumpFileReader.cpp" />
<ClCompile Include="..\..\CDVD\CompressedFileReader.cpp" />
<ClCompile Include="..\..\CDVD\OutputIsoFile.cpp" />
<ClInclude Include="..\..\CDVD\zlib_indexed.c" />
<ClCompile Include="..\..\DebugTools\Breakpoints.cpp" />
<ClCompile Include="..\..\DebugTools\DebugInterface.cpp" />
<ClCompile Include="..\..\DebugTools\DisassemblyManager.cpp" />

View File

@ -852,7 +852,14 @@
</ClCompile>
<ClCompile Include="..\..\gui\Debugger\DebuggerLists.cpp">
<Filter>AppHost\Debugger</Filter>
</ClCompile> </ItemGroup>
</ClCompile>
<ClCompile Include="..\..\CDVD\CompressedFileReader.cpp">
<Filter>System\ISO</Filter>
</ClCompile>
<ClCompile Include="..\..\CDVD\zlib_indexed.c">
<Filter>System\ISO</Filter>
</ClCompile>
</ItemGroup>
<ItemGroup>
<ClInclude Include="..\..\Patch.h">
<Filter>Misc</Filter>
@ -1262,7 +1269,8 @@
</ClInclude>
<ClInclude Include="..\..\gui\Debugger\DebuggerLists.h">
<Filter>AppHost\Debugger</Filter>
</ClInclude> </ItemGroup>
</ClInclude>
</ItemGroup>
<ItemGroup>
<ResourceCompile Include="..\..\..\3rdparty\wxWidgets\include\wx\msw\wx.rc">
<Filter>AppHost\Resources</Filter>