diff --git a/CMakeLists.txt b/CMakeLists.txt index 528b1cbb17..75656a7a38 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -516,6 +516,13 @@ include_directories(Source/Core) add_subdirectory(Externals/Bochs_disasm) include_directories(Externals/Bochs_disasm) +if(NOT XXHASH_FOUND) + message("Using static xxhash from Externals") + add_subdirectory(Externals/xxhash) + include_directories(Externals/xxhash) +endif() +LIST(APPEND LIBS xxhash) + # If zlib has already been found on a previous run of cmake don't check again # as the check seems to take a long time. if(NOT ZLIB_FOUND) diff --git a/Externals/xxhash/CMakeLists.txt b/Externals/xxhash/CMakeLists.txt new file mode 100644 index 0000000000..475d2ca43a --- /dev/null +++ b/Externals/xxhash/CMakeLists.txt @@ -0,0 +1,7 @@ +project(xxhash C) + +set(SRCS + xxhash.c +) + +add_library(xxhash STATIC ${SRCS}) diff --git a/Externals/xxhash/LICENSE b/Externals/xxhash/LICENSE new file mode 100644 index 0000000000..7de801ed1b --- /dev/null +++ b/Externals/xxhash/LICENSE @@ -0,0 +1,24 @@ +xxHash Library +Copyright (c) 2012-2014, Yann Collet +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, this + list of conditions and the following disclaimer in the documentation and/or + other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/Externals/xxhash/xxhash.c b/Externals/xxhash/xxhash.c new file mode 100644 index 0000000000..24a64b5f80 --- /dev/null +++ b/Externals/xxhash/xxhash.c @@ -0,0 +1,928 @@ +/* +xxHash - Fast Hash algorithm +Copyright (C) 2012-2014, Yann Collet. +BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +* Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +* Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +You can contact the author at : +- xxHash source repository : http://code.google.com/p/xxhash/ +- public discussion board : https://groups.google.com/forum/#!forum/lz4c +*/ + + +//************************************** +// Tuning parameters +//************************************** +// Unaligned memory access is automatically enabled for "common" CPU, such as x86. +// For others CPU, the compiler will be more cautious, and insert extra code to ensure aligned access is respected. +// If you know your target CPU supports unaligned memory access, you want to force this option manually to improve performance. +// You can also enable this parameter if you know your input data will always be aligned (boundaries of 4, for U32). +#if defined(__ARM_FEATURE_UNALIGNED) || defined(__i386) || defined(_M_IX86) || defined(__x86_64__) || defined(_M_X64) +# define XXH_USE_UNALIGNED_ACCESS 1 +#endif + +// XXH_ACCEPT_NULL_INPUT_POINTER : +// If the input pointer is a null pointer, xxHash default behavior is to trigger a memory access error, since it is a bad pointer. +// When this option is enabled, xxHash output for null input pointers will be the same as a null-length input. +// This option has a very small performance cost (only measurable on small inputs). +// By default, this option is disabled. To enable it, uncomment below define : +// #define XXH_ACCEPT_NULL_INPUT_POINTER 1 + +// XXH_FORCE_NATIVE_FORMAT : +// By default, xxHash library provides endian-independant Hash values, based on little-endian convention. +// Results are therefore identical for little-endian and big-endian CPU. +// This comes at a performance cost for big-endian CPU, since some swapping is required to emulate little-endian format. +// Should endian-independance be of no importance for your application, you may set the #define below to 1. +// It will improve speed for Big-endian CPU. +// This option has no impact on Little_Endian CPU. +#define XXH_FORCE_NATIVE_FORMAT 0 + +//************************************** +// Compiler Specific Options +//************************************** +// Disable some Visual warning messages +#ifdef _MSC_VER // Visual Studio +# pragma warning(disable : 4127) // disable: C4127: conditional expression is constant +#endif + +#ifdef _MSC_VER // Visual Studio +# define FORCE_INLINE static __forceinline +#else +# ifdef __GNUC__ +# define FORCE_INLINE static inline __attribute__((always_inline)) +# else +# define FORCE_INLINE static inline +# endif +#endif + +//************************************** +// Includes & Memory related functions +//************************************** +#include "xxhash.h" +// Modify the local functions below should you wish to use some other memory routines +// for malloc(), free() +#include +static void* XXH_malloc(size_t s) { return malloc(s); } +static void XXH_free (void* p) { free(p); } +// for memcpy() +#include +static void* XXH_memcpy(void* dest, const void* src, size_t size) +{ + return memcpy(dest,src,size); +} + + +//************************************** +// Basic Types +//************************************** +#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L // C99 +# include +typedef uint8_t BYTE; +typedef uint16_t U16; +typedef uint32_t U32; +typedef int32_t S32; +typedef uint64_t U64; +#else +typedef unsigned char BYTE; +typedef unsigned short U16; +typedef unsigned int U32; +typedef signed int S32; +typedef unsigned long long U64; +#endif + +#if defined(__GNUC__) && !defined(XXH_USE_UNALIGNED_ACCESS) +# define _PACKED __attribute__ ((packed)) +#else +# define _PACKED +#endif + +#if !defined(XXH_USE_UNALIGNED_ACCESS) && !defined(__GNUC__) +# ifdef __IBMC__ +# pragma pack(1) +# else +# pragma pack(push, 1) +# endif +#endif + +typedef struct _U32_S +{ + U32 v; +} _PACKED U32_S; +typedef struct _U64_S +{ + U64 v; +} _PACKED U64_S; + +#if !defined(XXH_USE_UNALIGNED_ACCESS) && !defined(__GNUC__) +# pragma pack(pop) +#endif + +#define A32(x) (((U32_S *)(x))->v) +#define A64(x) (((U64_S *)(x))->v) + + +//*************************************** +// Compiler-specific Functions and Macros +//*************************************** +#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) + +// Note : although _rotl exists for minGW (GCC under windows), performance seems poor +#if defined(_MSC_VER) +# define XXH_rotl32(x,r) _rotl(x,r) +# define XXH_rotl64(x,r) _rotl64(x,r) +#else +# define XXH_rotl32(x,r) ((x << r) | (x >> (32 - r))) +# define XXH_rotl64(x,r) ((x << r) | (x >> (64 - r))) +#endif + +#if defined(_MSC_VER) // Visual Studio +# define XXH_swap32 _byteswap_ulong +# define XXH_swap64 _byteswap_uint64 +#elif GCC_VERSION >= 403 +# define XXH_swap32 __builtin_bswap32 +# define XXH_swap64 __builtin_bswap64 +#else +static inline U32 XXH_swap32 (U32 x) +{ + return ((x << 24) & 0xff000000 ) | + ((x << 8) & 0x00ff0000 ) | + ((x >> 8) & 0x0000ff00 ) | + ((x >> 24) & 0x000000ff ); +} +static inline U64 XXH_swap64 (U64 x) +{ + return ((x << 56) & 0xff00000000000000ULL) | + ((x << 40) & 0x00ff000000000000ULL) | + ((x << 24) & 0x0000ff0000000000ULL) | + ((x << 8) & 0x000000ff00000000ULL) | + ((x >> 8) & 0x00000000ff000000ULL) | + ((x >> 24) & 0x0000000000ff0000ULL) | + ((x >> 40) & 0x000000000000ff00ULL) | + ((x >> 56) & 0x00000000000000ffULL); +} +#endif + + +//************************************** +// Constants +//************************************** +#define PRIME32_1 2654435761U +#define PRIME32_2 2246822519U +#define PRIME32_3 3266489917U +#define PRIME32_4 668265263U +#define PRIME32_5 374761393U + +#define PRIME64_1 11400714785074694791ULL +#define PRIME64_2 14029467366897019727ULL +#define PRIME64_3 1609587929392839161ULL +#define PRIME64_4 9650029242287828579ULL +#define PRIME64_5 2870177450012600261ULL + +//************************************** +// Architecture Macros +//************************************** +typedef enum { XXH_bigEndian=0, XXH_littleEndian=1 } XXH_endianess; +#ifndef XXH_CPU_LITTLE_ENDIAN // It is possible to define XXH_CPU_LITTLE_ENDIAN externally, for example using a compiler switch +static const int one = 1; +# define XXH_CPU_LITTLE_ENDIAN (*(char*)(&one)) +#endif + + +//************************************** +// Macros +//************************************** +#define XXH_STATIC_ASSERT(c) { enum { XXH_static_assert = 1/(!!(c)) }; } // use only *after* variable declarations + + +//**************************** +// Memory reads +//**************************** +typedef enum { XXH_aligned, XXH_unaligned } XXH_alignment; + +FORCE_INLINE U32 XXH_readLE32_align(const void* ptr, XXH_endianess endian, XXH_alignment align) +{ + if (align==XXH_unaligned) + return endian==XXH_littleEndian ? A32(ptr) : XXH_swap32(A32(ptr)); + else + return endian==XXH_littleEndian ? *(U32*)ptr : XXH_swap32(*(U32*)ptr); +} + +FORCE_INLINE U32 XXH_readLE32(const void* ptr, XXH_endianess endian) +{ + return XXH_readLE32_align(ptr, endian, XXH_unaligned); +} + +FORCE_INLINE U64 XXH_readLE64_align(const void* ptr, XXH_endianess endian, XXH_alignment align) +{ + if (align==XXH_unaligned) + return endian==XXH_littleEndian ? A64(ptr) : XXH_swap64(A64(ptr)); + else + return endian==XXH_littleEndian ? *(U64*)ptr : XXH_swap64(*(U64*)ptr); +} + +FORCE_INLINE U64 XXH_readLE64(const void* ptr, XXH_endianess endian) +{ + return XXH_readLE64_align(ptr, endian, XXH_unaligned); +} + + +//**************************** +// Simple Hash Functions +//**************************** +FORCE_INLINE U32 XXH32_endian_align(const void* input, size_t len, U32 seed, XXH_endianess endian, XXH_alignment align) +{ + const BYTE* p = (const BYTE*)input; + const BYTE* bEnd = p + len; + U32 h32; +#define XXH_get32bits(p) XXH_readLE32_align(p, endian, align) + +#ifdef XXH_ACCEPT_NULL_INPUT_POINTER + if (p==NULL) + { + len=0; + bEnd=p=(const BYTE*)(size_t)16; + } +#endif + + if (len>=16) + { + const BYTE* const limit = bEnd - 16; + U32 v1 = seed + PRIME32_1 + PRIME32_2; + U32 v2 = seed + PRIME32_2; + U32 v3 = seed + 0; + U32 v4 = seed - PRIME32_1; + + do + { + v1 += XXH_get32bits(p) * PRIME32_2; + v1 = XXH_rotl32(v1, 13); + v1 *= PRIME32_1; + p+=4; + v2 += XXH_get32bits(p) * PRIME32_2; + v2 = XXH_rotl32(v2, 13); + v2 *= PRIME32_1; + p+=4; + v3 += XXH_get32bits(p) * PRIME32_2; + v3 = XXH_rotl32(v3, 13); + v3 *= PRIME32_1; + p+=4; + v4 += XXH_get32bits(p) * PRIME32_2; + v4 = XXH_rotl32(v4, 13); + v4 *= PRIME32_1; + p+=4; + } + while (p<=limit); + + h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7) + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18); + } + else + { + h32 = seed + PRIME32_5; + } + + h32 += (U32) len; + + while (p+4<=bEnd) + { + h32 += XXH_get32bits(p) * PRIME32_3; + h32 = XXH_rotl32(h32, 17) * PRIME32_4 ; + p+=4; + } + + while (p> 15; + h32 *= PRIME32_2; + h32 ^= h32 >> 13; + h32 *= PRIME32_3; + h32 ^= h32 >> 16; + + return h32; +} + + +unsigned int XXH32 (const void* input, size_t len, unsigned seed) +{ +#if 0 + // Simple version, good for code maintenance, but unfortunately slow for small inputs + XXH32_state_t state; + XXH32_reset(&state, seed); + XXH32_update(&state, input, len); + return XXH32_digest(&state); +#else + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + +# if !defined(XXH_USE_UNALIGNED_ACCESS) + if ((((size_t)input) & 3) == 0) // Input is aligned, let's leverage the speed advantage + { + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned); + else + return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned); + } +# endif + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned); + else + return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned); +#endif +} + +FORCE_INLINE U64 XXH64_endian_align(const void* input, size_t len, U64 seed, XXH_endianess endian, XXH_alignment align) +{ + const BYTE* p = (const BYTE*)input; + const BYTE* bEnd = p + len; + U64 h64; +#define XXH_get64bits(p) XXH_readLE64_align(p, endian, align) + +#ifdef XXH_ACCEPT_NULL_INPUT_POINTER + if (p==NULL) + { + len=0; + bEnd=p=(const BYTE*)(size_t)32; + } +#endif + + if (len>=32) + { + const BYTE* const limit = bEnd - 32; + U64 v1 = seed + PRIME64_1 + PRIME64_2; + U64 v2 = seed + PRIME64_2; + U64 v3 = seed + 0; + U64 v4 = seed - PRIME64_1; + + do + { + v1 += XXH_get64bits(p) * PRIME64_2; + p+=8; + v1 = XXH_rotl64(v1, 31); + v1 *= PRIME64_1; + v2 += XXH_get64bits(p) * PRIME64_2; + p+=8; + v2 = XXH_rotl64(v2, 31); + v2 *= PRIME64_1; + v3 += XXH_get64bits(p) * PRIME64_2; + p+=8; + v3 = XXH_rotl64(v3, 31); + v3 *= PRIME64_1; + v4 += XXH_get64bits(p) * PRIME64_2; + p+=8; + v4 = XXH_rotl64(v4, 31); + v4 *= PRIME64_1; + } + while (p<=limit); + + h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18); + + v1 *= PRIME64_2; + v1 = XXH_rotl64(v1, 31); + v1 *= PRIME64_1; + h64 ^= v1; + h64 = h64 * PRIME64_1 + PRIME64_4; + + v2 *= PRIME64_2; + v2 = XXH_rotl64(v2, 31); + v2 *= PRIME64_1; + h64 ^= v2; + h64 = h64 * PRIME64_1 + PRIME64_4; + + v3 *= PRIME64_2; + v3 = XXH_rotl64(v3, 31); + v3 *= PRIME64_1; + h64 ^= v3; + h64 = h64 * PRIME64_1 + PRIME64_4; + + v4 *= PRIME64_2; + v4 = XXH_rotl64(v4, 31); + v4 *= PRIME64_1; + h64 ^= v4; + h64 = h64 * PRIME64_1 + PRIME64_4; + } + else + { + h64 = seed + PRIME64_5; + } + + h64 += (U64) len; + + while (p+8<=bEnd) + { + U64 k1 = XXH_get64bits(p); + k1 *= PRIME64_2; + k1 = XXH_rotl64(k1,31); + k1 *= PRIME64_1; + h64 ^= k1; + h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4; + p+=8; + } + + if (p+4<=bEnd) + { + h64 ^= (U64)(XXH_get32bits(p)) * PRIME64_1; + h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3; + p+=4; + } + + while (p> 33; + h64 *= PRIME64_2; + h64 ^= h64 >> 29; + h64 *= PRIME64_3; + h64 ^= h64 >> 32; + + return h64; +} + + +unsigned long long XXH64 (const void* input, size_t len, unsigned long long seed) +{ +#if 0 + // Simple version, good for code maintenance, but unfortunately slow for small inputs + XXH64_state_t state; + XXH64_reset(&state, seed); + XXH64_update(&state, input, len); + return XXH64_digest(&state); +#else + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + +# if !defined(XXH_USE_UNALIGNED_ACCESS) + if ((((size_t)input) & 7)==0) // Input is aligned, let's leverage the speed advantage + { + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned); + else + return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned); + } +# endif + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned); + else + return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned); +#endif +} + +/**************************************************** + * Advanced Hash Functions +****************************************************/ + +/*** Allocation ***/ +typedef struct +{ + U64 total_len; + U32 seed; + U32 v1; + U32 v2; + U32 v3; + U32 v4; + U32 mem32[4]; /* defined as U32 for alignment */ + U32 memsize; +} XXH_istate32_t; + +typedef struct +{ + U64 total_len; + U64 seed; + U64 v1; + U64 v2; + U64 v3; + U64 v4; + U64 mem64[4]; /* defined as U64 for alignment */ + U32 memsize; +} XXH_istate64_t; + + +XXH32_state_t* XXH32_createState(void) +{ + XXH_STATIC_ASSERT(sizeof(XXH32_state_t) >= sizeof(XXH_istate32_t)); // A compilation error here means XXH32_state_t is not large enough + return (XXH32_state_t*)XXH_malloc(sizeof(XXH32_state_t)); +} +XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr) +{ + XXH_free(statePtr); + return XXH_OK; +}; + +XXH64_state_t* XXH64_createState(void) +{ + XXH_STATIC_ASSERT(sizeof(XXH64_state_t) >= sizeof(XXH_istate64_t)); // A compilation error here means XXH64_state_t is not large enough + return (XXH64_state_t*)XXH_malloc(sizeof(XXH64_state_t)); +} +XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr) +{ + XXH_free(statePtr); + return XXH_OK; +}; + + +/*** Hash feed ***/ + +XXH_errorcode XXH32_reset(XXH32_state_t* state_in, U32 seed) +{ + XXH_istate32_t* state = (XXH_istate32_t*) state_in; + state->seed = seed; + state->v1 = seed + PRIME32_1 + PRIME32_2; + state->v2 = seed + PRIME32_2; + state->v3 = seed + 0; + state->v4 = seed - PRIME32_1; + state->total_len = 0; + state->memsize = 0; + return XXH_OK; +} + +XXH_errorcode XXH64_reset(XXH64_state_t* state_in, unsigned long long seed) +{ + XXH_istate64_t* state = (XXH_istate64_t*) state_in; + state->seed = seed; + state->v1 = seed + PRIME64_1 + PRIME64_2; + state->v2 = seed + PRIME64_2; + state->v3 = seed + 0; + state->v4 = seed - PRIME64_1; + state->total_len = 0; + state->memsize = 0; + return XXH_OK; +} + + +FORCE_INLINE XXH_errorcode XXH32_update_endian (XXH32_state_t* state_in, const void* input, size_t len, XXH_endianess endian) +{ + XXH_istate32_t* state = (XXH_istate32_t *) state_in; + const BYTE* p = (const BYTE*)input; + const BYTE* const bEnd = p + len; + +#ifdef XXH_ACCEPT_NULL_INPUT_POINTER + if (input==NULL) return XXH_ERROR; +#endif + + state->total_len += len; + + if (state->memsize + len < 16) // fill in tmp buffer + { + XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, len); + state->memsize += (U32)len; + return XXH_OK; + } + + if (state->memsize) // some data left from previous update + { + XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, 16-state->memsize); + { + const U32* p32 = state->mem32; + state->v1 += XXH_readLE32(p32, endian) * PRIME32_2; + state->v1 = XXH_rotl32(state->v1, 13); + state->v1 *= PRIME32_1; + p32++; + state->v2 += XXH_readLE32(p32, endian) * PRIME32_2; + state->v2 = XXH_rotl32(state->v2, 13); + state->v2 *= PRIME32_1; + p32++; + state->v3 += XXH_readLE32(p32, endian) * PRIME32_2; + state->v3 = XXH_rotl32(state->v3, 13); + state->v3 *= PRIME32_1; + p32++; + state->v4 += XXH_readLE32(p32, endian) * PRIME32_2; + state->v4 = XXH_rotl32(state->v4, 13); + state->v4 *= PRIME32_1; + p32++; + } + p += 16-state->memsize; + state->memsize = 0; + } + + if (p <= bEnd-16) + { + const BYTE* const limit = bEnd - 16; + U32 v1 = state->v1; + U32 v2 = state->v2; + U32 v3 = state->v3; + U32 v4 = state->v4; + + do + { + v1 += XXH_readLE32(p, endian) * PRIME32_2; + v1 = XXH_rotl32(v1, 13); + v1 *= PRIME32_1; + p+=4; + v2 += XXH_readLE32(p, endian) * PRIME32_2; + v2 = XXH_rotl32(v2, 13); + v2 *= PRIME32_1; + p+=4; + v3 += XXH_readLE32(p, endian) * PRIME32_2; + v3 = XXH_rotl32(v3, 13); + v3 *= PRIME32_1; + p+=4; + v4 += XXH_readLE32(p, endian) * PRIME32_2; + v4 = XXH_rotl32(v4, 13); + v4 *= PRIME32_1; + p+=4; + } + while (p<=limit); + + state->v1 = v1; + state->v2 = v2; + state->v3 = v3; + state->v4 = v4; + } + + if (p < bEnd) + { + XXH_memcpy(state->mem32, p, bEnd-p); + state->memsize = (int)(bEnd-p); + } + + return XXH_OK; +} + +XXH_errorcode XXH32_update (XXH32_state_t* state_in, const void* input, size_t len) +{ + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH32_update_endian(state_in, input, len, XXH_littleEndian); + else + return XXH32_update_endian(state_in, input, len, XXH_bigEndian); +} + + + +FORCE_INLINE U32 XXH32_digest_endian (const XXH32_state_t* state_in, XXH_endianess endian) +{ + XXH_istate32_t* state = (XXH_istate32_t*) state_in; + const BYTE * p = (const BYTE*)state->mem32; + BYTE* bEnd = (BYTE*)(state->mem32) + state->memsize; + U32 h32; + + if (state->total_len >= 16) + { + h32 = XXH_rotl32(state->v1, 1) + XXH_rotl32(state->v2, 7) + XXH_rotl32(state->v3, 12) + XXH_rotl32(state->v4, 18); + } + else + { + h32 = state->seed + PRIME32_5; + } + + h32 += (U32) state->total_len; + + while (p+4<=bEnd) + { + h32 += XXH_readLE32(p, endian) * PRIME32_3; + h32 = XXH_rotl32(h32, 17) * PRIME32_4; + p+=4; + } + + while (p> 15; + h32 *= PRIME32_2; + h32 ^= h32 >> 13; + h32 *= PRIME32_3; + h32 ^= h32 >> 16; + + return h32; +} + + +U32 XXH32_digest (const XXH32_state_t* state_in) +{ + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH32_digest_endian(state_in, XXH_littleEndian); + else + return XXH32_digest_endian(state_in, XXH_bigEndian); +} + + +FORCE_INLINE XXH_errorcode XXH64_update_endian (XXH64_state_t* state_in, const void* input, size_t len, XXH_endianess endian) +{ + XXH_istate64_t * state = (XXH_istate64_t *) state_in; + const BYTE* p = (const BYTE*)input; + const BYTE* const bEnd = p + len; + +#ifdef XXH_ACCEPT_NULL_INPUT_POINTER + if (input==NULL) return XXH_ERROR; +#endif + + state->total_len += len; + + if (state->memsize + len < 32) // fill in tmp buffer + { + XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, len); + state->memsize += (U32)len; + return XXH_OK; + } + + if (state->memsize) // some data left from previous update + { + XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, 32-state->memsize); + { + const U64* p64 = state->mem64; + state->v1 += XXH_readLE64(p64, endian) * PRIME64_2; + state->v1 = XXH_rotl64(state->v1, 31); + state->v1 *= PRIME64_1; + p64++; + state->v2 += XXH_readLE64(p64, endian) * PRIME64_2; + state->v2 = XXH_rotl64(state->v2, 31); + state->v2 *= PRIME64_1; + p64++; + state->v3 += XXH_readLE64(p64, endian) * PRIME64_2; + state->v3 = XXH_rotl64(state->v3, 31); + state->v3 *= PRIME64_1; + p64++; + state->v4 += XXH_readLE64(p64, endian) * PRIME64_2; + state->v4 = XXH_rotl64(state->v4, 31); + state->v4 *= PRIME64_1; + p64++; + } + p += 32-state->memsize; + state->memsize = 0; + } + + if (p+32 <= bEnd) + { + const BYTE* const limit = bEnd - 32; + U64 v1 = state->v1; + U64 v2 = state->v2; + U64 v3 = state->v3; + U64 v4 = state->v4; + + do + { + v1 += XXH_readLE64(p, endian) * PRIME64_2; + v1 = XXH_rotl64(v1, 31); + v1 *= PRIME64_1; + p+=8; + v2 += XXH_readLE64(p, endian) * PRIME64_2; + v2 = XXH_rotl64(v2, 31); + v2 *= PRIME64_1; + p+=8; + v3 += XXH_readLE64(p, endian) * PRIME64_2; + v3 = XXH_rotl64(v3, 31); + v3 *= PRIME64_1; + p+=8; + v4 += XXH_readLE64(p, endian) * PRIME64_2; + v4 = XXH_rotl64(v4, 31); + v4 *= PRIME64_1; + p+=8; + } + while (p<=limit); + + state->v1 = v1; + state->v2 = v2; + state->v3 = v3; + state->v4 = v4; + } + + if (p < bEnd) + { + XXH_memcpy(state->mem64, p, bEnd-p); + state->memsize = (int)(bEnd-p); + } + + return XXH_OK; +} + +XXH_errorcode XXH64_update (XXH64_state_t* state_in, const void* input, size_t len) +{ + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH64_update_endian(state_in, input, len, XXH_littleEndian); + else + return XXH64_update_endian(state_in, input, len, XXH_bigEndian); +} + + + +FORCE_INLINE U64 XXH64_digest_endian (const XXH64_state_t* state_in, XXH_endianess endian) +{ + XXH_istate64_t * state = (XXH_istate64_t *) state_in; + const BYTE * p = (const BYTE*)state->mem64; + BYTE* bEnd = (BYTE*)state->mem64 + state->memsize; + U64 h64; + + if (state->total_len >= 32) + { + U64 v1 = state->v1; + U64 v2 = state->v2; + U64 v3 = state->v3; + U64 v4 = state->v4; + + h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18); + + v1 *= PRIME64_2; + v1 = XXH_rotl64(v1, 31); + v1 *= PRIME64_1; + h64 ^= v1; + h64 = h64*PRIME64_1 + PRIME64_4; + + v2 *= PRIME64_2; + v2 = XXH_rotl64(v2, 31); + v2 *= PRIME64_1; + h64 ^= v2; + h64 = h64*PRIME64_1 + PRIME64_4; + + v3 *= PRIME64_2; + v3 = XXH_rotl64(v3, 31); + v3 *= PRIME64_1; + h64 ^= v3; + h64 = h64*PRIME64_1 + PRIME64_4; + + v4 *= PRIME64_2; + v4 = XXH_rotl64(v4, 31); + v4 *= PRIME64_1; + h64 ^= v4; + h64 = h64*PRIME64_1 + PRIME64_4; + } + else + { + h64 = state->seed + PRIME64_5; + } + + h64 += (U64) state->total_len; + + while (p+8<=bEnd) + { + U64 k1 = XXH_readLE64(p, endian); + k1 *= PRIME64_2; + k1 = XXH_rotl64(k1,31); + k1 *= PRIME64_1; + h64 ^= k1; + h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4; + p+=8; + } + + if (p+4<=bEnd) + { + h64 ^= (U64)(XXH_readLE32(p, endian)) * PRIME64_1; + h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3; + p+=4; + } + + while (p> 33; + h64 *= PRIME64_2; + h64 ^= h64 >> 29; + h64 *= PRIME64_3; + h64 ^= h64 >> 32; + + return h64; +} + + +unsigned long long XXH64_digest (const XXH64_state_t* state_in) +{ + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH64_digest_endian(state_in, XXH_littleEndian); + else + return XXH64_digest_endian(state_in, XXH_bigEndian); +} + + diff --git a/Externals/xxhash/xxhash.h b/Externals/xxhash/xxhash.h new file mode 100644 index 0000000000..55b45015a4 --- /dev/null +++ b/Externals/xxhash/xxhash.h @@ -0,0 +1,156 @@ +/* + xxHash - Extremely Fast Hash algorithm + Header File + Copyright (C) 2012-2014, Yann Collet. + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - xxHash source repository : http://code.google.com/p/xxhash/ +*/ + +/* Notice extracted from xxHash homepage : + +xxHash is an extremely fast Hash algorithm, running at RAM speed limits. +It also successfully passes all tests from the SMHasher suite. + +Comparison (single thread, Windows Seven 32 bits, using SMHasher on a Core 2 Duo @3GHz) + +Name Speed Q.Score Author +xxHash 5.4 GB/s 10 +CrapWow 3.2 GB/s 2 Andrew +MumurHash 3a 2.7 GB/s 10 Austin Appleby +SpookyHash 2.0 GB/s 10 Bob Jenkins +SBox 1.4 GB/s 9 Bret Mulvey +Lookup3 1.2 GB/s 9 Bob Jenkins +SuperFastHash 1.2 GB/s 1 Paul Hsieh +CityHash64 1.05 GB/s 10 Pike & Alakuijala +FNV 0.55 GB/s 5 Fowler, Noll, Vo +CRC32 0.43 GB/s 9 +MD5-32 0.33 GB/s 10 Ronald L. Rivest +SHA1-32 0.28 GB/s 10 + +Q.Score is a measure of quality of the hash function. +It depends on successfully passing SMHasher test set. +10 is a perfect score. +*/ + +#pragma once + +#if defined (__cplusplus) +extern "C" { +#endif + + +/***************************** + Includes +*****************************/ +#include /* size_t */ + + +/***************************** + Type +*****************************/ +typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode; + + + +/***************************** + Simple Hash Functions +*****************************/ + +unsigned int XXH32 (const void* input, size_t length, unsigned seed); +unsigned long long XXH64 (const void* input, size_t length, unsigned long long seed); + +/* +XXH32() : + Calculate the 32-bits hash of sequence "length" bytes stored at memory address "input". + The memory between input & input+length must be valid (allocated and read-accessible). + "seed" can be used to alter the result predictably. + This function successfully passes all SMHasher tests. + Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark) : 5.4 GB/s +XXH64() : + Calculate the 64-bits hash of sequence of length "len" stored at memory address "input". +*/ + + + +/***************************** + Advanced Hash Functions +*****************************/ +typedef struct { long long ll[ 6]; } XXH32_state_t; +typedef struct { long long ll[11]; } XXH64_state_t; + +/* +These structures allow static allocation of XXH states. +States must then be initialized using XXHnn_reset() before first use. + +If you prefer dynamic allocation, please refer to functions below. +*/ + +XXH32_state_t* XXH32_createState(void); +XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr); + +XXH64_state_t* XXH64_createState(void); +XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr); + +/* +These functions create and release memory for XXH state. +States must then be initialized using XXHnn_reset() before first use. +*/ + + +XXH_errorcode XXH32_reset (XXH32_state_t* statePtr, unsigned seed); +XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void* input, size_t length); +unsigned int XXH32_digest (const XXH32_state_t* statePtr); + +XXH_errorcode XXH64_reset (XXH64_state_t* statePtr, unsigned long long seed); +XXH_errorcode XXH64_update (XXH64_state_t* statePtr, const void* input, size_t length); +unsigned long long XXH64_digest (const XXH64_state_t* statePtr); + +/* +These functions calculate the xxHash of an input provided in multiple smaller packets, +as opposed to an input provided as a single block. + +XXH state space must first be allocated, using either static or dynamic method provided above. + +Start a new hash by initializing state with a seed, using XXHnn_reset(). + +Then, feed the hash state by calling XXHnn_update() as many times as necessary. +Obviously, input must be valid, meaning allocated and read accessible. +The function returns an error code, with 0 meaning OK, and any other value meaning there is an error. + +Finally, you can produce a hash anytime, by using XXHnn_digest(). +This function returns the final nn-bits hash. +You can nonetheless continue feeding the hash state with more input, +and therefore get some new hashes, by calling again XXHnn_digest(). + +When you are done, don't forget to free XXH state space, using typically XXHnn_freeState(). +*/ + + +#if defined (__cplusplus) +} +#endif diff --git a/Externals/xxhash/xxhash.vcxproj b/Externals/xxhash/xxhash.vcxproj new file mode 100644 index 0000000000..55574759a8 --- /dev/null +++ b/Externals/xxhash/xxhash.vcxproj @@ -0,0 +1,61 @@ + + + + + Debug + Win32 + + + Debug + x64 + + + Release + Win32 + + + Release + x64 + + + + {677EA016-1182-440C-9345-DC88D1E98C0C} + + + + StaticLibrary + v120 + Unicode + + + true + + + false + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Externals/xxhash/xxhsum.c b/Externals/xxhash/xxhsum.c new file mode 100644 index 0000000000..e090e51118 --- /dev/null +++ b/Externals/xxhash/xxhsum.c @@ -0,0 +1,689 @@ +/* +bench.c - Demo program to benchmark open-source algorithm +Copyright (C) Yann Collet 2012-2014 + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 2 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along +with this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +You can contact the author at : +- Blog homepage : http://fastcompression.blogspot.com/ +- Discussion group : https://groups.google.com/forum/?fromgroups#!forum/lz4c +*/ + +/************************************** + * Compiler Options + *************************************/ +/* MS Visual */ +#if defined(_MSC_VER) || defined(_WIN32) +# define _CRT_SECURE_NO_WARNINGS /* removes visual warnings */ +# define BMK_LEGACY_TIMER 1 /* gettimeofday() not supported by MSVC */ +#endif + +/* Under Linux at least, pull in the *64 commands */ +#define _LARGEFILE64_SOURCE + + +/************************************** + * Includes + *************************************/ +#include // malloc +#include // fprintf, fopen, ftello64, fread, stdin, stdout; when present : _fileno +#include // strcmp +#include // stat64 +#include // stat64 + +#include "xxhash.h" + + +/************************************** + * OS-Specific Includes + *************************************/ +// Use ftime() if gettimeofday() is not available on your target +#if defined(BMK_LEGACY_TIMER) +# include // timeb, ftime +#else +# include // gettimeofday +#endif + +#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__) +# include // _O_BINARY +# include // _setmode, _isatty +# ifdef __MINGW32__ + int _fileno(FILE *stream); // MINGW somehow forgets to include this windows declaration into +# endif +# define SET_BINARY_MODE(file) _setmode(_fileno(file), _O_BINARY) +# define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream)) +#else +# include // isatty, STDIN_FILENO +# define SET_BINARY_MODE(file) +# define IS_CONSOLE(stdStream) isatty(STDIN_FILENO) +#endif + +#if !defined(S_ISREG) +# define S_ISREG(x) (((x) & S_IFMT) == S_IFREG) +#endif + + +/************************************** + * Basic Types + *************************************/ +#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L // C99 +# include + typedef uint8_t BYTE; + typedef uint16_t U16; + typedef uint32_t U32; + typedef int32_t S32; + typedef uint64_t U64; +#else + typedef unsigned char BYTE; + typedef unsigned short U16; + typedef unsigned int U32; + typedef signed int S32; + typedef unsigned long long U64; +#endif + + +/************************************** + * Constants + *************************************/ +#define PROGRAM_NAME exename +#define PROGRAM_VERSION "" +#define COMPILED __DATE__ +#define AUTHOR "Yann Collet" +#define WELCOME_MESSAGE "*** %s %i-bits %s, by %s (%s) ***\n", PROGRAM_NAME, (int)(sizeof(void*)*8), PROGRAM_VERSION, AUTHOR, COMPILED + +#define NBLOOPS 3 // Default number of benchmark iterations +#define TIMELOOP 2500 // Minimum timing per iteration +#define PRIME 2654435761U + +#define KB *(1<<10) +#define MB *(1<<20) +#define GB *(1U<<30) + +#define MAX_MEM (2 GB - 64 MB) + +static const char stdinName[] = "-"; + + +//************************************** +// Display macros +//************************************** +#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) +#define DISPLAYRESULT(...) fprintf(stdout, __VA_ARGS__) +#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) DISPLAY(__VA_ARGS__); +static unsigned g_displayLevel = 1; + + +//************************************** +// Unit variables +//************************************** +static int g_nbIterations = NBLOOPS; +static int g_fn_selection = 1; // required within main() & usage() + + +//********************************************************* +// Benchmark Functions +//********************************************************* + +#if defined(BMK_LEGACY_TIMER) + +static int BMK_GetMilliStart(void) +{ + // Based on Legacy ftime() + // Rolls over every ~ 12.1 days (0x100000/24/60/60) + // Use GetMilliSpan to correct for rollover + struct timeb tb; + int nCount; + ftime( &tb ); + nCount = (int) (tb.millitm + (tb.time & 0xfffff) * 1000); + return nCount; +} + +#else + +static int BMK_GetMilliStart(void) +{ + // Based on newer gettimeofday() + // Use GetMilliSpan to correct for rollover + struct timeval tv; + int nCount; + gettimeofday(&tv, NULL); + nCount = (int) (tv.tv_usec/1000 + (tv.tv_sec & 0xfffff) * 1000); + return nCount; +} + +#endif + +static int BMK_GetMilliSpan( int nTimeStart ) +{ + int nSpan = BMK_GetMilliStart() - nTimeStart; + if ( nSpan < 0 ) + nSpan += 0x100000 * 1000; + return nSpan; +} + + +static size_t BMK_findMaxMem(U64 requestedMem) +{ + size_t step = (64 MB); + size_t allocatedMemory; + BYTE* testmem=NULL; + + requestedMem += 3*step; + requestedMem -= (size_t)requestedMem & (step-1); + if (requestedMem > MAX_MEM) requestedMem = MAX_MEM; + allocatedMemory = (size_t)requestedMem; + + while (!testmem) + { + allocatedMemory -= step; + testmem = (BYTE*) malloc((size_t)allocatedMemory); + } + free (testmem); + + return (size_t) (allocatedMemory - step); +} + + +static U64 BMK_GetFileSize(char* infilename) +{ + int r; +#if defined(_MSC_VER) + struct _stat64 statbuf; + r = _stat64(infilename, &statbuf); +#else + struct stat statbuf; + r = stat(infilename, &statbuf); +#endif + if (r || !S_ISREG(statbuf.st_mode)) return 0; // No good... + return (U64)statbuf.st_size; +} + + +static int BMK_benchFile(char** fileNamesTable, int nbFiles) +{ + int fileIdx=0; + U32 hashResult=0; + + U64 totals = 0; + double totalc = 0.; + + + // Loop for each file + while (fileIdx inFileSize) benchedSize = (size_t)inFileSize; + if (benchedSize < inFileSize) + { + DISPLAY("Not enough memory for '%s' full size; testing %i MB only...\n", inFileName, (int)(benchedSize>>20)); + } + + buffer = (char*)malloc((size_t )benchedSize+16); + if(!buffer) + { + DISPLAY("\nError: not enough memory!\n"); + fclose(inFile); + return 12; + } + alignedBuffer = (buffer+15) - (((size_t)(buffer+15)) & 0xF); // align on next 16 bytes boundaries + + // Fill input buffer + DISPLAY("\rLoading %s... \n", inFileName); + readSize = fread(alignedBuffer, 1, benchedSize, inFile); + fclose(inFile); + + if(readSize != benchedSize) + { + DISPLAY("\nError: problem reading file '%s' !! \n", inFileName); + free(buffer); + return 13; + } + + + // Bench XXH32 + { + int interationNb; + double fastestC = 100000000.; + + DISPLAY("\r%79s\r", ""); // Clean display line + for (interationNb = 1; interationNb <= g_nbIterations; interationNb++) + { + int nbHashes = 0; + int milliTime; + + DISPLAY("%1i-%-14.14s : %10i ->\r", interationNb, "XXH32", (int)benchedSize); + + // Hash loop + milliTime = BMK_GetMilliStart(); + while(BMK_GetMilliStart() == milliTime); + milliTime = BMK_GetMilliStart(); + while(BMK_GetMilliSpan(milliTime) < TIMELOOP) + { + int i; + for (i=0; i<100; i++) + { + hashResult = XXH32(alignedBuffer, benchedSize, 0); + nbHashes++; + } + } + milliTime = BMK_GetMilliSpan(milliTime); + if ((double)milliTime < fastestC*nbHashes) fastestC = (double)milliTime/nbHashes; + DISPLAY("%1i-%-14.14s : %10i -> %7.1f MB/s\r", interationNb, "XXH32", (int)benchedSize, (double)benchedSize / fastestC / 1000.); + } + DISPLAY("%-16.16s : %10i -> %7.1f MB/s 0x%08X\n", "XXH32", (int)benchedSize, (double)benchedSize / fastestC / 1000., hashResult); + + totals += benchedSize; + totalc += fastestC; + } + + // Bench Unaligned XXH32 + { + int interationNb; + double fastestC = 100000000.; + + DISPLAY("\r%79s\r", ""); // Clean display line + for (interationNb = 1; (interationNb <= g_nbIterations) && ((benchedSize>1)); interationNb++) + { + int nbHashes = 0; + int milliTime; + + DISPLAY("%1i-%-14.14s : %10i ->\r", interationNb, "(unaligned)", (int)benchedSize); + // Hash loop + milliTime = BMK_GetMilliStart(); + while(BMK_GetMilliStart() == milliTime); + milliTime = BMK_GetMilliStart(); + while(BMK_GetMilliSpan(milliTime) < TIMELOOP) + { + int i; + for (i=0; i<100; i++) + { + hashResult = XXH32(alignedBuffer+1, benchedSize-1, 0); + nbHashes++; + } + } + milliTime = BMK_GetMilliSpan(milliTime); + if ((double)milliTime < fastestC*nbHashes) fastestC = (double)milliTime/nbHashes; + DISPLAY("%1i-%-14.14s : %10i -> %7.1f MB/s\r", interationNb, "XXH32 (unaligned)", (int)(benchedSize-1), (double)(benchedSize-1) / fastestC / 1000.); + } + DISPLAY("%-16.16s : %10i -> %7.1f MB/s \n", "XXH32 (unaligned)", (int)benchedSize-1, (double)(benchedSize-1) / fastestC / 1000.); + } + + // Bench XXH64 + { + int interationNb; + double fastestC = 100000000.; + unsigned long long h64 = 0; + + DISPLAY("\r%79s\r", ""); // Clean display line + for (interationNb = 1; interationNb <= g_nbIterations; interationNb++) + { + int nbHashes = 0; + int milliTime; + + DISPLAY("%1i-%-14.14s : %10i ->\r", interationNb, "XXH64", (int)benchedSize); + + // Hash loop + milliTime = BMK_GetMilliStart(); + while(BMK_GetMilliStart() == milliTime); + milliTime = BMK_GetMilliStart(); + while(BMK_GetMilliSpan(milliTime) < TIMELOOP) + { + int i; + for (i=0; i<100; i++) + { + h64 = XXH64(alignedBuffer, benchedSize, 0); + nbHashes++; + } + } + milliTime = BMK_GetMilliSpan(milliTime); + if ((double)milliTime < fastestC*nbHashes) fastestC = (double)milliTime/nbHashes; + DISPLAY("%1i-%-14.14s : %10i -> %7.1f MB/s\r", interationNb, "XXH64", (int)benchedSize, (double)benchedSize / fastestC / 1000.); + } + DISPLAY("%-16.16s : %10i -> %7.1f MB/s 0x%08X%08X\n", "XXH64", (int)benchedSize, (double)benchedSize / fastestC / 1000., (U32)(h64>>32), (U32)(h64)); + + totals += benchedSize; + totalc += fastestC; + } + + free(buffer); + } + + if (nbFiles > 1) + printf("%-16.16s :%11llu -> %7.1f MB/s\n", " TOTAL", (long long unsigned int)totals, (double)totals/totalc/1000.); + + return 0; +} + + + +static void BMK_checkResult(U32 r1, U32 r2) +{ + static int nbTests = 1; + + if (r1==r2) DISPLAY("\rTest%3i : %08X == %08X ok ", nbTests, r1, r2); + else + { + DISPLAY("\rERROR : Test%3i : %08X <> %08X !!!!! \n", nbTests, r1, r2); + exit(1); + } + nbTests++; +} + + +static void BMK_checkResult64(U64 r1, U64 r2) +{ + static int nbTests = 1; + + if (r1!=r2) + { + DISPLAY("\rERROR : Test%3i : 64-bits values non equals !!!!! \n", nbTests); + DISPLAY("\r %08X%08X != %08X%08X \n", (U32)(r1>>32), (U32)r1, (U32)(r2<<32), (U32)r2); + exit(1); + } + nbTests++; +} + + +static void BMK_testSequence64(void* sentence, int len, U64 seed, U64 Nresult) +{ + U64 Dresult; + XXH64_state_t state; + int index; + + Dresult = XXH64(sentence, len, seed); + BMK_checkResult64(Dresult, Nresult); + + XXH64_reset(&state, seed); + XXH64_update(&state, sentence, len); + Dresult = XXH64_digest(&state); + BMK_checkResult64(Dresult, Nresult); + + XXH64_reset(&state, seed); + for (index=0; index>24); + prime *= prime; + } + + BMK_testSequence(NULL, 0, 0, 0x02CC5D05); + BMK_testSequence(NULL, 0, PRIME, 0x36B78AE7); + BMK_testSequence(sanityBuffer, 1, 0, 0xB85CBEE5); + BMK_testSequence(sanityBuffer, 1, PRIME, 0xD5845D64); + BMK_testSequence(sanityBuffer, 14, 0, 0xE5AA0AB4); + BMK_testSequence(sanityBuffer, 14, PRIME, 0x4481951D); + BMK_testSequence(sanityBuffer, SANITY_BUFFER_SIZE, 0, 0x1F1AA412); + BMK_testSequence(sanityBuffer, SANITY_BUFFER_SIZE, PRIME, 0x498EC8E2); + + BMK_testSequence64(NULL , 0, 0, 0xEF46DB3751D8E999ULL); + BMK_testSequence64(NULL , 0, PRIME, 0xAC75FDA2929B17EFULL); + BMK_testSequence64(sanityBuffer, 1, 0, 0x4FCE394CC88952D8ULL); + BMK_testSequence64(sanityBuffer, 1, PRIME, 0x739840CB819FA723ULL); + BMK_testSequence64(sanityBuffer, 14, 0, 0xCFFA8DB881BC3A3DULL); + BMK_testSequence64(sanityBuffer, 14, PRIME, 0x5B9611585EFCC9CBULL); + BMK_testSequence64(sanityBuffer, SANITY_BUFFER_SIZE, 0, 0x0EAB543384F878ADULL); + BMK_testSequence64(sanityBuffer, SANITY_BUFFER_SIZE, PRIME, 0xCAA65939306F1E21ULL); + + DISPLAY("\r%79s\r", ""); // Clean display line + DISPLAYLEVEL(2, "Sanity check -- all tests ok\n"); +} + + +static int BMK_hash(const char* fileName, U32 hashNb) +{ + FILE* inFile; + size_t const blockSize = 64 KB; + size_t readSize; + char* buffer; + XXH64_state_t state; + + // Check file existence + if (fileName == stdinName) + { + inFile = stdin; + SET_BINARY_MODE(stdin); + } + else + inFile = fopen( fileName, "rb" ); + if (inFile==NULL) + { + DISPLAY( "Pb opening %s\n", fileName); + return 11; + } + + // Memory allocation & restrictions + buffer = (char*)malloc(blockSize); + if(!buffer) + { + DISPLAY("\nError: not enough memory!\n"); + fclose(inFile); + return 12; + } + + // Init + switch(hashNb) + { + case 0: + XXH32_reset((XXH32_state_t*)&state, 0); + break; + case 1: + XXH64_reset(&state, 0); + break; + default: + DISPLAY("Error : bad hash algorithm ID\n"); + fclose(inFile); + free(buffer); + return -1; + } + + + // Load file & update hash + DISPLAY("\rLoading %s... \r", fileName); + readSize = 1; + while (readSize) + { + readSize = fread(buffer, 1, blockSize, inFile); + switch(hashNb) + { + case 0: + XXH32_update((XXH32_state_t*)&state, buffer, readSize); + break; + case 1: + XXH64_update(&state, buffer, readSize); + break; + default: + break; + } + } + fclose(inFile); + free(buffer); + + // display Hash + switch(hashNb) + { + case 0: + { + U32 h32 = XXH32_digest((XXH32_state_t*)&state); + DISPLAYRESULT("%08x %s \n", h32, fileName); + break; + } + case 1: + { + U64 h64 = XXH64_digest(&state); + DISPLAYRESULT("%08x%08x %s \n", (U32)(h64>>32), (U32)(h64), fileName); + break; + } + default: + break; + } + + return 0; +} + + +//********************************************************* +// Main +//********************************************************* + +static int usage(const char* exename) +{ + DISPLAY( WELCOME_MESSAGE ); + DISPLAY( "Usage :\n"); + DISPLAY( " %s [arg] [filename]\n", exename); + DISPLAY( "When no filename provided, or - provided : use stdin as input\n"); + DISPLAY( "Arguments :\n"); + DISPLAY( " -H# : hash selection : 0=32bits, 1=64bits (default %i)\n", g_fn_selection); + DISPLAY( " -b : benchmark mode \n"); + DISPLAY( " -i# : number of iterations (benchmark mode; default %i)\n", g_nbIterations); + DISPLAY( " -h : help (this text)\n"); + return 0; +} + + +static int badusage(const char* exename) +{ + DISPLAY("Wrong parameters\n"); + usage(exename); + return 1; +} + + +int main(int argc, char** argv) +{ + int i, filenamesStart=0; + const char* input_filename = (char*)stdinName; + const char* exename = argv[0]; + U32 benchmarkMode = 0; + + // xxh32sum default to 32 bits checksum + if (strstr(exename, "xxh32sum")!=NULL) g_fn_selection=0; + + for(i=1; i 1) return badusage(exename); + + return BMK_hash(input_filename, g_fn_selection); +} diff --git a/Source/Core/VideoCommon/VideoCommon.vcxproj b/Source/Core/VideoCommon/VideoCommon.vcxproj index f76c59e8ba..649cd5095e 100644 --- a/Source/Core/VideoCommon/VideoCommon.vcxproj +++ b/Source/Core/VideoCommon/VideoCommon.vcxproj @@ -147,6 +147,9 @@ {b441cc62-877e-4b3f-93e0-0de80544f705} + + {677EA016-1182-440C-9345-DC88D1E98C0C} + {ff213b23-2c26-4214-9f88-85271e557e87} diff --git a/Source/VSProps/Base.props b/Source/VSProps/Base.props index 4a361b0cb7..3e447b5420 100644 --- a/Source/VSProps/Base.props +++ b/Source/VSProps/Base.props @@ -46,6 +46,7 @@ $(ExternalsDir)SOIL;%(AdditionalIncludeDirectories) $(ExternalsDir)wxWidgets3;%(AdditionalIncludeDirectories) $(ExternalsDir)wxWidgets3\Include;%(AdditionalIncludeDirectories) + $(ExternalsDir)xxhash;%(AdditionalIncludeDirectories) $(ExternalsDir)zlib;%(AdditionalIncludeDirectories) _CRT_SECURE_NO_WARNINGS;_CRT_SECURE_NO_DEPRECATE;%(PreprocessorDefinitions) USE_UPNP;%(PreprocessorDefinitions) diff --git a/Source/VSProps/QtCompile.props b/Source/VSProps/QtCompile.props index 8727c7d60c..9a21863036 100644 --- a/Source/VSProps/QtCompile.props +++ b/Source/VSProps/QtCompile.props @@ -89,7 +89,7 @@ in order to prevent the trailing slash from escaping the doublequote after value replacement. --> "-I$(QtIncludeDir)QtWidgets" "-I$(QtIncludeDir)QtGui" "-I$(QtIncludeDir)QtCore" "-I$(QtIncludeDir) " "-I$(QtToolOutDir) " -I. - "-I$(ExternalsDir)zlib" "-I$(ExternalsDir)SOIL" "-I$(ExternalsDir)SFML\include" "-I$(ExternalsDir)portaudio\include" "-I$(ExternalsDir)polarssl\include" "-I$(ExternalsDir)miniupnpc\src" "-I$(ExternalsDir)LZO" "-I$(ExternalsDir)libusbx\libusb" "-I$(ExternalsDir)libpng" "-I$(ExternalsDir)GL" "-I$(ExternalsDir)Bochs_disasm" "-I$(ExternalsDir) " "-I$(CoreDir) " $(MocIncludes) + "-I$(ExternalsDir)xxhash" "-I$(ExternalsDir)zlib" "-I$(ExternalsDir)SOIL" "-I$(ExternalsDir)SFML\include" "-I$(ExternalsDir)portaudio\include" "-I$(ExternalsDir)polarssl\include" "-I$(ExternalsDir)miniupnpc\src" "-I$(ExternalsDir)LZO" "-I$(ExternalsDir)libusbx\libusb" "-I$(ExternalsDir)libpng" "-I$(ExternalsDir)GL" "-I$(ExternalsDir)Bochs_disasm" "-I$(ExternalsDir) " "-I$(CoreDir) " $(MocIncludes) - \ No newline at end of file + diff --git a/Source/VSProps/WXWOverrides.props b/Source/VSProps/WXWOverrides.props index 213827577f..7b7c253709 100644 --- a/Source/VSProps/WXWOverrides.props +++ b/Source/VSProps/WXWOverrides.props @@ -10,11 +10,11 @@ Async 4267;%(DisableSpecificWarnings) __WXMSW__;WXBUILDING;%(PreprocessorDefinitions) - ..\..\;..\..\include;..\..\..\zlib;..\..\..\libpng + ..\..\;..\..\include;..\..\..\xxhash;..\..\..\zlib;..\..\..\libpng "/manifestdependency:type='Win32' name='Microsoft.Windows.Common-Controls' version='6.0.0.0' processorArchitecture='*' publicKeyToken='6595b64144ccf1df' language='*'\" %(AdditionalOptions) - \ No newline at end of file + diff --git a/Source/dolphin-emu.sln b/Source/dolphin-emu.sln index 7361f45c3c..c4fb3b6511 100644 --- a/Source/dolphin-emu.sln +++ b/Source/dolphin-emu.sln @@ -37,6 +37,8 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "png", "..\Externals\libpng\ EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "SOIL", "..\Externals\SOIL\SOIL.vcxproj", "{B441CC62-877E-4B3F-93E0-0DE80544F705}" EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "xxhash", "..\Externals\xxhash\xxhash.vcxproj", "{677EA016-1182-440C-9345-DC88D1E98C0C}" +EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "zlib", "..\Externals\zlib\zlib.vcxproj", "{FF213B23-2C26-4214-9F88-85271E557E87}" EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "SoundTouch", "..\Externals\soundtouch\SoundTouch.vcxproj", "{EC082900-B4D8-42E9-9663-77F02F6936AE}" @@ -133,6 +135,10 @@ Global {B441CC62-877E-4B3F-93E0-0DE80544F705}.Debug|x64.Build.0 = Debug|x64 {B441CC62-877E-4B3F-93E0-0DE80544F705}.Release|x64.ActiveCfg = Release|x64 {B441CC62-877E-4B3F-93E0-0DE80544F705}.Release|x64.Build.0 = Release|x64 + {677EA016-1182-440C-9345-DC88D1E98C0C}.Debug|x64.ActiveCfg = Debug|x64 + {677EA016-1182-440C-9345-DC88D1E98C0C}.Debug|x64.Build.0 = Debug|x64 + {677EA016-1182-440C-9345-DC88D1E98C0C}.Release|x64.ActiveCfg = Release|x64 + {677EA016-1182-440C-9345-DC88D1E98C0C}.Release|x64.Build.0 = Release|x64 {FF213B23-2C26-4214-9F88-85271E557E87}.Debug|x64.ActiveCfg = Debug|x64 {FF213B23-2C26-4214-9F88-85271E557E87}.Debug|x64.Build.0 = Debug|x64 {FF213B23-2C26-4214-9F88-85271E557E87}.Release|x64.ActiveCfg = Release|x64 @@ -206,6 +212,7 @@ Global {31643FDB-1BB8-4965-9DE7-000FC88D35AE} = {87ADDFF9-5768-4DA2-A33B-2477593D6677} {4C9F135B-A85E-430C-BAD4-4C67EF5FC12C} = {87ADDFF9-5768-4DA2-A33B-2477593D6677} {B441CC62-877E-4B3F-93E0-0DE80544F705} = {87ADDFF9-5768-4DA2-A33B-2477593D6677} + {677EA016-1182-440C-9345-DC88D1E98C0C} = {87ADDFF9-5768-4DA2-A33B-2477593D6677} {FF213B23-2C26-4214-9F88-85271E557E87} = {87ADDFF9-5768-4DA2-A33B-2477593D6677} {EC082900-B4D8-42E9-9663-77F02F6936AE} = {87ADDFF9-5768-4DA2-A33B-2477593D6677} {1C8436C9-DBAF-42BE-83BC-CF3EC9175ABE} = {87ADDFF9-5768-4DA2-A33B-2477593D6677}