Merge pull request #1785 from degasus/custom_texture

VideoCommon: Custom texture handling

Source/Core/Common/Hash.cpp

@@ -510,19 +510,15 @@ u64 GetHash64(const u8 *src, int len, u32 samples)
 }
 
 // sets the hash function used for the texture cache
-void SetHash64Function(bool useHiresTextures)
+void SetHash64Function()
 {
-	if (useHiresTextures)
-	{
-		ptrHashFunction = &GetHashHiresTexture;
-	}
 #if _M_SSE >= 0x402
-	else if (cpu_info.bSSE4_2 && !useHiresTextures) // sse crc32 version
+	if (cpu_info.bSSE4_2) // sse crc32 version
 	{
 		ptrHashFunction = &GetCRC32;
 	}
-#endif
 	else
+#endif
 	{
 		ptrHashFunction = &GetMurmurHash3;
 	}

Source/Core/Common/Hash.h

@@ -16,4 +16,4 @@ u64 GetCRC32(const u8 *src, int len, u32 samples);   // SSE4.2 version of CRC32
 u64 GetHashHiresTexture(const u8 *src, int len, u32 samples);
 u64 GetMurmurHash3(const u8 *src, int len, u32 samples);
 u64 GetHash64(const u8 *src, int len, u32 samples);
-void SetHash64Function(bool useHiresTextures);
+void SetHash64Function();

Source/Core/VideoCommon/HiresTextures.cpp

@@ -13,14 +13,14 @@
 #include "Common/FileUtil.h"
 #include "Common/StringUtil.h"
 
+#include "Core/ConfigManager.h"
+
 #include "VideoCommon/HiresTextures.h"
+#include "VideoCommon/VideoConfig.h"
 
-namespace HiresTextures
+std::unordered_map<std::string, std::string> HiresTexture::textureMap;
-{
 
-static std::map<std::string, std::string> textureMap;
+void HiresTexture::Init(const std::string& gameCode)
-
-void Init(const std::string& gameCode)
 {
 	textureMap.clear();
 
@@ -80,85 +80,92 @@ void Init(const std::string& gameCode)
 	}
 }
 
-bool HiresTexExists(const std::string& filename)
+std::string HiresTexture::GenBaseName(const u8* texture, size_t texture_size, const u8* tlut, size_t tlut_size, u32 width, u32 height, int format)
 {
-	return textureMap.find(filename) != textureMap.end();
+	u64 tex_hash = GetHashHiresTexture(texture, (int)texture_size, g_ActiveConfig.iSafeTextureCache_ColorSamples);
+	u64 tlut_hash = 0;
+	u64 hash = tex_hash;
+	if (tlut_size)
+	{
+		tlut_hash = GetHashHiresTexture(tlut, (int)tlut_size, g_ActiveConfig.iSafeTextureCache_ColorSamples);
+		hash ^= tlut_hash;
+	}
+	return StringFromFormat("%s_%08x_%i", SConfig::GetInstance().m_LocalCoreStartupParameter.m_strUniqueID.c_str(), (u32)hash, (u16)format);
 }
 
-PC_TexFormat GetHiresTex(const std::string& filename, unsigned int* pWidth, unsigned int* pHeight, unsigned int* required_size, int texformat, unsigned int data_size, u8* data)
+HiresTexture* HiresTexture::Search(const u8* texture, size_t texture_size, const u8* tlut, size_t tlut_size, u32 width, u32 height, int format)
 {
-	if (textureMap.find(filename) == textureMap.end())
+	std::string base_filename = GenBaseName(texture, texture_size, tlut, tlut_size, width, height, format);
-		return PC_TEX_FMT_NONE;
 
-	int width;
+	HiresTexture* ret = nullptr;
-	int height;
+	for (int level = 0;; level++)
-	int channels;
-
-	File::IOFile file;
-	file.Open(textureMap[filename], "rb");
-	std::vector<u8> buffer(file.GetSize());
-	file.ReadBytes(buffer.data(), file.GetSize());
-
-	u8* temp = SOIL_load_image_from_memory(buffer.data(), (int)buffer.size(), &width, &height, &channels, SOIL_LOAD_RGBA);
-
-	if (temp == nullptr)
 	{
-		ERROR_LOG(VIDEO, "Custom texture %s failed to load", textureMap[filename].c_str());
+		std::string filename = base_filename;
-		return PC_TEX_FMT_NONE;
+		if (level)
-	}
-
-	*pWidth = width;
-	*pHeight = height;
-
-	//int offset = 0;
-	PC_TexFormat returnTex = PC_TEX_FMT_NONE;
-
-	// TODO(neobrain): This function currently has no way to enforce RGBA32
-	// output, which however is required on some configurations to function
-	// properly. As a lazy workaround, we hence disable the optimized code
-	// path for now.
-#if 0
-	switch (texformat)
-	{
-	case GX_TF_I4:
-	case GX_TF_I8:
-	case GX_TF_IA4:
-	case GX_TF_IA8:
-		*required_size = width * height * 8;
-		if (data_size < *required_size)
-			goto cleanup;
-
-		for (int i = 0; i < width * height * 4; i += 4)
 		{
-			// Rather than use a luminosity function, just use the most intense color for luminance
+			filename += StringFromFormat("_mip%u", level);
-			// TODO(neobrain): Isn't this kind of.. stupid?
-			data[offset++] = *std::max_element(temp+i, temp+i+3);
-			data[offset++] = temp[i+3];
 		}
-		returnTex = PC_TEX_FMT_IA8;
-		break;
-	default:
-		*required_size = width * height * 4;
-		if (data_size < *required_size)
-			goto cleanup;
 
-		memcpy(data, temp, width * height * 4);
+		if (textureMap.find(filename) != textureMap.end())
-		returnTex = PC_TEX_FMT_RGBA32;
+		{
-		break;
+			Level l;
+
+			File::IOFile file;
+			file.Open(textureMap[filename], "rb");
+			std::vector<u8> buffer(file.GetSize());
+			file.ReadBytes(buffer.data(), file.GetSize());
+
+			int channels;
+			l.data = SOIL_load_image_from_memory(buffer.data(), (int)buffer.size(), (int*)&l.width, (int*)&l.height, &channels, SOIL_LOAD_RGBA);
+			l.data_size = (size_t)l.width * l.height * 4;
+
+			if (l.data == nullptr)
+			{
+				ERROR_LOG(VIDEO, "Custom texture %s failed to load", filename.c_str());
+				break;
+			}
+
+			if (!level)
+			{
+				if (l.width * height != l.height * width)
+					ERROR_LOG(VIDEO, "Invalid custom texture size %dx%d for texture %s. The aspect differs from the native size %dx%d.",
+					          l.width, l.height, filename.c_str(), width, height);
+				if (l.width % width || l.height % height)
+					WARN_LOG(VIDEO, "Invalid custom texture size %dx%d for texture %s. Please use an integer upscaling factor based on the native size %dx%d.",
+					         l.width, l.height, filename.c_str(), width, height);
+				width = l.width;
+				height = l.height;
+			}
+			else if (width != l.width || height != l.height)
+			{
+				ERROR_LOG(VIDEO, "Invalid custom texture size %dx%d for texture %s. This mipmap layer _must_ be %dx%d.",
+				          l.width, l.height, filename.c_str(), width, height);
+				SOIL_free_image_data(l.data);
+				break;
+			}
+
+			// calculate the size of the next mipmap
+			width >>= 1;
+			height >>= 1;
+
+			if (!ret)
+				ret = new HiresTexture();
+			ret->m_levels.push_back(l);
+		}
+		else
+		{
+			break;
+		}
 	}
-#else
-	*required_size = width * height * 4;
-	if (data_size < *required_size)
-		goto cleanup;
 
-	memcpy(data, temp, width * height * 4);
+	return ret;
-	returnTex = PC_TEX_FMT_RGBA32;
-#endif
-
-	INFO_LOG(VIDEO, "Loading custom texture from %s", textureMap[filename].c_str());
-cleanup:
-	SOIL_free_image_data(temp);
-	return returnTex;
 }
 
+HiresTexture::~HiresTexture()
+{
+	for (auto& l : m_levels)
+	{
+		SOIL_free_image_data(l.data);
+	}
 }
+

Source/Core/VideoCommon/HiresTextures.h

@@ -4,15 +4,43 @@
 
 #pragma once
 
-#include <map>
 #include <string>
+#include <unordered_map>
 #include "VideoCommon/TextureDecoder.h"
 #include "VideoCommon/VideoCommon.h"
 
-namespace HiresTextures
+class HiresTexture
 {
-void Init(const std::string& gameCode);
+public:
-bool HiresTexExists(const std::string& filename);
+	static void Init(const std::string& gameCode);
-PC_TexFormat GetHiresTex(const std::string& fileName, unsigned int* pWidth, unsigned int* pHeight, unsigned int* required_size, int texformat, unsigned int data_size, u8* data);
 
-}
+	static HiresTexture* Search(
+		const u8* texture, size_t texture_size,
+		const u8* tlut, size_t tlut_size,
+		u32 width, u32 height,
+		int format
+	);
+
+	static std::string GenBaseName(
+		const u8* texture, size_t texture_size,
+		const u8* tlut, size_t tlut_size,
+		u32 width, u32 height,
+		int format
+	);
+
+	~HiresTexture();
+
+	struct Level
+	{
+		u8* data;
+		size_t data_size;
+		u32 width, height;
+	};
+	std::vector<Level> m_levels;
+
+	static std::unordered_map<std::string, std::string> textureMap;
+
+private:
+	HiresTexture() {}
+
+};

Source/Core/VideoCommon/TextureCacheBase.cpp

@@ -28,7 +28,7 @@ enum
 TextureCache *g_texture_cache;
 
 GC_ALIGNED16(u8 *TextureCache::temp) = nullptr;
-unsigned int TextureCache::temp_size;
+size_t TextureCache::temp_size;
 
 TextureCache::TexCache TextureCache::textures;
 TextureCache::RenderTargetPool TextureCache::render_target_pool;
@@ -41,6 +41,16 @@ TextureCache::TCacheEntryBase::~TCacheEntryBase()
 {
 }
 
+void TextureCache::CheckTempSize(size_t required_size)
+{
+	if (required_size <= temp_size)
+		return;
+
+	temp_size = required_size;
+	FreeAlignedMemory(temp);
+	temp = (u8*)AllocateAlignedMemory(temp_size, 16);
+}
+
 TextureCache::TextureCache()
 {
 	temp_size = 2048 * 2048 * 4;
@@ -50,9 +60,9 @@ TextureCache::TextureCache()
 	TexDecoder_SetTexFmtOverlayOptions(g_ActiveConfig.bTexFmtOverlayEnable, g_ActiveConfig.bTexFmtOverlayCenter);
 
 	if (g_ActiveConfig.bHiresTextures && !g_ActiveConfig.bDumpTextures)
-		HiresTextures::Init(SConfig::GetInstance().m_LocalCoreStartupParameter.m_strUniqueID);
+		HiresTexture::Init(SConfig::GetInstance().m_LocalCoreStartupParameter.m_strUniqueID);
 
-	SetHash64Function(g_ActiveConfig.bHiresTextures || g_ActiveConfig.bDumpTextures);
+	SetHash64Function();
 
 	invalidate_texture_cache_requested = false;
 }
@@ -98,9 +108,8 @@ void TextureCache::OnConfigChanged(VideoConfig& config)
 			g_texture_cache->Invalidate();
 
 			if (g_ActiveConfig.bHiresTextures)
-				HiresTextures::Init(SConfig::GetInstance().m_LocalCoreStartupParameter.m_strUniqueID);
+				HiresTexture::Init(SConfig::GetInstance().m_LocalCoreStartupParameter.m_strUniqueID);
 
-			SetHash64Function(g_ActiveConfig.bHiresTextures || g_ActiveConfig.bDumpTextures);
 			TexDecoder_SetTexFmtOverlayOptions(g_ActiveConfig.bTexFmtOverlayEnable, g_ActiveConfig.bTexFmtOverlayCenter);
 
 			invalidate_texture_cache_requested = false;
@@ -253,72 +262,8 @@ void TextureCache::ClearRenderTargets()
 	}
 }
 
-bool TextureCache::CheckForCustomTextureLODs(u64 tex_hash, int texformat, unsigned int levels)
+void TextureCache::DumpTexture(TCacheEntryBase* entry, std::string basename, unsigned int level)
 {
-	if (levels == 1)
-		return false;
-
-	// Just checking if the necessary files exist, if they can't be loaded or have incorrect dimensions LODs will be black
-	std::string texBasePathTemp = StringFromFormat("%s_%08x_%i", SConfig::GetInstance().m_LocalCoreStartupParameter.m_strUniqueID.c_str(), (u32) (tex_hash & 0x00000000FFFFFFFFLL), texformat);
-
-	for (unsigned int level = 1; level < levels; ++level)
-	{
-		std::string texPathTemp = StringFromFormat("%s_mip%u", texBasePathTemp.c_str(), level);
-		if (!HiresTextures::HiresTexExists(texPathTemp))
-		{
-			if (level > 1)
-				WARN_LOG(VIDEO, "Couldn't find custom texture LOD with index %u (filename: %s), disabling custom LODs for this texture", level, texPathTemp.c_str());
-
-			return false;
-		}
-	}
-
-	return true;
-}
-
-PC_TexFormat TextureCache::LoadCustomTexture(u64 tex_hash, int texformat, unsigned int level, unsigned int* widthp, unsigned int* heightp)
-{
-	std::string texPathTemp;
-	unsigned int newWidth = 0;
-	unsigned int newHeight = 0;
-	u32 tex_hash_u32 = tex_hash & 0x00000000FFFFFFFFLL;
-
-	if (level == 0)
-		texPathTemp = StringFromFormat("%s_%08x_%i", SConfig::GetInstance().m_LocalCoreStartupParameter.m_strUniqueID.c_str(), tex_hash_u32, texformat);
-	else
-		texPathTemp = StringFromFormat("%s_%08x_%i_mip%u", SConfig::GetInstance().m_LocalCoreStartupParameter.m_strUniqueID.c_str(), tex_hash_u32, texformat, level);
-
-	unsigned int required_size = 0;
-	PC_TexFormat ret = HiresTextures::GetHiresTex(texPathTemp, &newWidth, &newHeight, &required_size, texformat, temp_size, temp);
-	if (ret == PC_TEX_FMT_NONE && temp_size < required_size)
-	{
-		// Allocate more memory and try again
-		// TODO: Should probably check if newWidth and newHeight are texture dimensions which are actually supported by the current video backend
-		temp_size = required_size;
-		FreeAlignedMemory(temp);
-		temp = (u8*)AllocateAlignedMemory(temp_size, 16);
-		ret = HiresTextures::GetHiresTex(texPathTemp, &newWidth, &newHeight, &required_size, texformat, temp_size, temp);
-	}
-
-	if (ret != PC_TEX_FMT_NONE)
-	{
-		unsigned int width = *widthp, height = *heightp;
-		if (level > 0 && (newWidth != width || newHeight != height))
-			ERROR_LOG(VIDEO, "Invalid custom texture size %dx%d for texture %s. This mipmap layer _must_ be %dx%d.", newWidth, newHeight, texPathTemp.c_str(), width, height);
-		if (newWidth * height != newHeight * width)
-			ERROR_LOG(VIDEO, "Invalid custom texture size %dx%d for texture %s. The aspect differs from the native size %dx%d.", newWidth, newHeight, texPathTemp.c_str(), width, height);
-		if (newWidth % width || newHeight % height)
-			WARN_LOG(VIDEO, "Invalid custom texture size %dx%d for texture %s. Please use an integer upscaling factor based on the native size %dx%d.", newWidth, newHeight, texPathTemp.c_str(), width, height);
-
-		*widthp = newWidth;
-		*heightp = newHeight;
-	}
-	return ret;
-}
-
-void TextureCache::DumpTexture(TCacheEntryBase* entry, unsigned int level)
-{
-	std::string filename;
 	std::string szDir = File::GetUserPath(D_DUMPTEXTURES_IDX) +
 		SConfig::GetInstance().m_LocalCoreStartupParameter.m_strUniqueID;
 
@@ -326,20 +271,11 @@ void TextureCache::DumpTexture(TCacheEntryBase* entry, unsigned int level)
 	if (!File::Exists(szDir) || !File::IsDirectory(szDir))
 		File::CreateDir(szDir);
 
-	// For compatibility with old texture packs, don't print the LOD index for level 0.
+	if (level > 0)
-	 // TODO: TLUT format should actually be stored in filename? :/
-	if (level == 0)
 	{
-		filename = StringFromFormat("%s/%s_%08x_%i.png", szDir.c_str(),
+		basename += StringFromFormat("_mip%i", level);
-			SConfig::GetInstance().m_LocalCoreStartupParameter.m_strUniqueID.c_str(),
-			(u32)(entry->hash & 0x00000000FFFFFFFFLL), entry->format & 0xFFFF);
-	}
-	else
-	{
-		filename = StringFromFormat("%s/%s_%08x_%i_mip%i.png", szDir.c_str(),
-				SConfig::GetInstance().m_LocalCoreStartupParameter.m_strUniqueID.c_str(),
-				(u32) (entry->hash & 0x00000000FFFFFFFFLL), entry->format & 0xFFFF, level);
 	}
+	std::string filename = szDir + "/" + basename + ".png";
 
 	if (!File::Exists(filename))
 		entry->Save(filename, level);
@@ -399,9 +335,10 @@ TextureCache::TCacheEntryBase* TextureCache::Load(unsigned int const stage,
 
 	// TODO: This doesn't hash GB tiles for preloaded RGBA8 textures (instead, it's hashing more data from the low tmem bank than it should)
 	tex_hash = GetHash64(src_data, texture_size, g_ActiveConfig.iSafeTextureCache_ColorSamples);
+	u32 palette_size = 0;
 	if (isPaletteTexture)
 	{
-		const u32 palette_size = TexDecoder_GetPaletteSize(texformat);
+		palette_size = TexDecoder_GetPaletteSize(texformat);
 		tlut_hash = GetHash64(&texMem[tlutaddr], palette_size, g_ActiveConfig.iSafeTextureCache_ColorSamples);
 
 		// NOTE: For non-paletted textures, texID is equal to the texture address.
@@ -473,17 +410,23 @@ TextureCache::TCacheEntryBase* TextureCache::Load(unsigned int const stage,
 		}
 	}
 
-	bool using_custom_texture = false;
+	std::unique_ptr<HiresTexture> hires_tex;
-
 	if (g_ActiveConfig.bHiresTextures)
 	{
-		pcfmt = LoadCustomTexture(tex_hash, texformat, 0, &width, &height);
+		hires_tex.reset(HiresTexture::Search(
-		if (pcfmt != PC_TEX_FMT_NONE)
+			src_data, texture_size,
+			&texMem[tlutaddr], palette_size,
+			width, height,
+			texformat
+		));
+
+		if (hires_tex)
 		{
-			if (expandedWidth != width || expandedHeight != height)
+			auto& l = hires_tex->m_levels[0];
+			if (l.width != width || l.height != height)
 			{
-				expandedWidth = width;
+				width = l.width;
-				expandedHeight = height;
+				height = l.height;
 
 				// If we thought we could reuse the texture before, make sure to pool it now!
 				if (entry)
@@ -492,11 +435,15 @@ TextureCache::TCacheEntryBase* TextureCache::Load(unsigned int const stage,
 					entry = nullptr;
 				}
 			}
-			using_custom_texture = true;
+			expandedWidth = l.width;
+			expandedHeight = l.height;
+			CheckTempSize(l.data_size);
+			memcpy(temp, l.data, l.data_size);
+			pcfmt = PC_TEX_FMT_RGBA32;
 		}
 	}
 
-	if (!using_custom_texture)
+	if (!hires_tex)
 	{
 		if (!(texformat == GX_TF_RGBA8 && from_tmem))
 		{
@@ -511,7 +458,7 @@ TextureCache::TCacheEntryBase* TextureCache::Load(unsigned int const stage,
 	}
 
 	u32 texLevels = use_mipmaps ? (maxlevel + 1) : 1;
-	const bool using_custom_lods = using_custom_texture && CheckForCustomTextureLODs(tex_hash, texformat, texLevels);
+	const bool using_custom_lods = hires_tex && hires_tex->m_levels.size() >= texLevels;
 	// Only load native mips if their dimensions fit to our virtual texture dimensions
 	const bool use_native_mips = use_mipmaps && !using_custom_lods && (width == nativeW && height == nativeH);
 	texLevels = (use_native_mips || using_custom_lods) ? texLevels : 1; // TODO: Should be forced to 1 for non-pow2 textures (e.g. efb copies with automatically adjusted IR)
@@ -549,8 +496,17 @@ TextureCache::TCacheEntryBase* TextureCache::Load(unsigned int const stage,
 	else
 		entry->type = TCET_NORMAL;
 
-	if (g_ActiveConfig.bDumpTextures && !using_custom_texture)
+	std::string basename = "";
-		DumpTexture(entry, 0);
+	if (g_ActiveConfig.bDumpTextures && !hires_tex)
+	{
+		basename = HiresTexture::GenBaseName(
+			src_data, texture_size,
+			&texMem[tlutaddr], palette_size,
+			width, height,
+			texformat
+		);
+		DumpTexture(entry, basename, 0);
+	}
 
 	u32 level = 1;
 	// load mips - TODO: Loading mipmaps from tmem is untested!
@@ -585,18 +541,17 @@ TextureCache::TCacheEntryBase* TextureCache::Load(unsigned int const stage,
 				entry->Load(mip_width, mip_height, expanded_mip_width, level);
 
 				if (g_ActiveConfig.bDumpTextures)
-					DumpTexture(entry, level);
+					DumpTexture(entry, basename, level);
 			}
 		}
 		else if (using_custom_lods)
 		{
 			for (; level != texLevels; ++level)
 			{
-				unsigned int mip_width = CalculateLevelSize(width, level);
+				auto& l = hires_tex->m_levels[level];
-				unsigned int mip_height = CalculateLevelSize(height, level);
+				CheckTempSize(l.data_size);
-
+				memcpy(temp, l.data, l.data_size);
-				LoadCustomTexture(tex_hash, texformat, level, &mip_width, &mip_height);
+				entry->Load(l.width, l.height, l.width, level);
-				entry->Load(mip_width, mip_height, mip_width, level);
 			}
 		}
 	}

Source/Core/VideoCommon/TextureCacheBase.h

@@ -116,13 +116,12 @@ public:
 protected:
 	TextureCache();
 
-	static  GC_ALIGNED16(u8 *temp);
+	static GC_ALIGNED16(u8 *temp);
-	static unsigned int temp_size;
+	static size_t temp_size;
 
 private:
-	static bool CheckForCustomTextureLODs(u64 tex_hash, int texformat, unsigned int levels);
+	static void DumpTexture(TCacheEntryBase* entry, std::string basename, unsigned int level);
-	static PC_TexFormat LoadCustomTexture(u64 tex_hash, int texformat, unsigned int level, unsigned int* width, unsigned int* height);
+	static void CheckTempSize(size_t required_size);
-	static void DumpTexture(TCacheEntryBase* entry, unsigned int level);
 
 	static TCacheEntryBase* AllocateRenderTarget(unsigned int width, unsigned int height);
 	static void FreeRenderTarget(TCacheEntryBase* entry);