/* bench.c - Demo program to benchmark open-source algorithm Copyright (C) Yann Collet 2012-2014 This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. You can contact the author at : - Blog homepage : http://fastcompression.blogspot.com/ - Discussion group : https://groups.google.com/forum/?fromgroups#!forum/lz4c */ /************************************** * Compiler Options *************************************/ /* MS Visual */ #if defined(_MSC_VER) || defined(_WIN32) # define _CRT_SECURE_NO_WARNINGS /* removes visual warnings */ # define BMK_LEGACY_TIMER 1 /* gettimeofday() not supported by MSVC */ #endif /* Under Linux at least, pull in the *64 commands */ #define _LARGEFILE64_SOURCE /************************************** * Includes *************************************/ #include // malloc #include // fprintf, fopen, ftello64, fread, stdin, stdout; when present : _fileno #include // strcmp #include // stat64 #include // stat64 #include "xxhash.h" /************************************** * OS-Specific Includes *************************************/ // Use ftime() if gettimeofday() is not available on your target #if defined(BMK_LEGACY_TIMER) # include // timeb, ftime #else # include // gettimeofday #endif #if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__) # include // _O_BINARY # include // _setmode, _isatty # ifdef __MINGW32__ int _fileno(FILE *stream); // MINGW somehow forgets to include this windows declaration into # endif # define SET_BINARY_MODE(file) _setmode(_fileno(file), _O_BINARY) # define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream)) #else # include // isatty, STDIN_FILENO # define SET_BINARY_MODE(file) # define IS_CONSOLE(stdStream) isatty(STDIN_FILENO) #endif #if !defined(S_ISREG) # define S_ISREG(x) (((x) & S_IFMT) == S_IFREG) #endif /************************************** * Basic Types *************************************/ #if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L // C99 # include typedef uint8_t BYTE; typedef uint16_t U16; typedef uint32_t U32; typedef int32_t S32; typedef uint64_t U64; #else typedef unsigned char BYTE; typedef unsigned short U16; typedef unsigned int U32; typedef signed int S32; typedef unsigned long long U64; #endif /************************************** * Constants *************************************/ #define PROGRAM_NAME exename #define PROGRAM_VERSION "" #define COMPILED __DATE__ #define AUTHOR "Yann Collet" #define WELCOME_MESSAGE "*** %s %i-bits %s, by %s (%s) ***\n", PROGRAM_NAME, (int)(sizeof(void*)*8), PROGRAM_VERSION, AUTHOR, COMPILED #define NBLOOPS 3 // Default number of benchmark iterations #define TIMELOOP 2500 // Minimum timing per iteration #define PRIME 2654435761U #define KB *(1<<10) #define MB *(1<<20) #define GB *(1U<<30) #define MAX_MEM (2 GB - 64 MB) static const char stdinName[] = "-"; //************************************** // Display macros //************************************** #define DISPLAY(...) fprintf(stderr, __VA_ARGS__) #define DISPLAYRESULT(...) fprintf(stdout, __VA_ARGS__) #define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) DISPLAY(__VA_ARGS__); static unsigned g_displayLevel = 1; //************************************** // Unit variables //************************************** static int g_nbIterations = NBLOOPS; static int g_fn_selection = 1; // required within main() & usage() //********************************************************* // Benchmark Functions //********************************************************* #if defined(BMK_LEGACY_TIMER) static int BMK_GetMilliStart(void) { // Based on Legacy ftime() // Rolls over every ~ 12.1 days (0x100000/24/60/60) // Use GetMilliSpan to correct for rollover struct timeb tb; int nCount; ftime( &tb ); nCount = (int) (tb.millitm + (tb.time & 0xfffff) * 1000); return nCount; } #else static int BMK_GetMilliStart(void) { // Based on newer gettimeofday() // Use GetMilliSpan to correct for rollover struct timeval tv; int nCount; gettimeofday(&tv, NULL); nCount = (int) (tv.tv_usec/1000 + (tv.tv_sec & 0xfffff) * 1000); return nCount; } #endif static int BMK_GetMilliSpan( int nTimeStart ) { int nSpan = BMK_GetMilliStart() - nTimeStart; if ( nSpan < 0 ) nSpan += 0x100000 * 1000; return nSpan; } static size_t BMK_findMaxMem(U64 requestedMem) { size_t step = (64 MB); size_t allocatedMemory; BYTE* testmem=NULL; requestedMem += 3*step; requestedMem -= (size_t)requestedMem & (step-1); if (requestedMem > MAX_MEM) requestedMem = MAX_MEM; allocatedMemory = (size_t)requestedMem; while (!testmem) { allocatedMemory -= step; testmem = (BYTE*) malloc((size_t)allocatedMemory); } free (testmem); return (size_t) (allocatedMemory - step); } static U64 BMK_GetFileSize(char* infilename) { int r; #if defined(_MSC_VER) struct _stat64 statbuf; r = _stat64(infilename, &statbuf); #else struct stat statbuf; r = stat(infilename, &statbuf); #endif if (r || !S_ISREG(statbuf.st_mode)) return 0; // No good... return (U64)statbuf.st_size; } static int BMK_benchFile(char** fileNamesTable, int nbFiles) { int fileIdx=0; U32 hashResult=0; U64 totals = 0; double totalc = 0.; // Loop for each file while (fileIdx inFileSize) benchedSize = (size_t)inFileSize; if (benchedSize < inFileSize) { DISPLAY("Not enough memory for '%s' full size; testing %i MB only...\n", inFileName, (int)(benchedSize>>20)); } buffer = (char*)malloc((size_t )benchedSize+16); if(!buffer) { DISPLAY("\nError: not enough memory!\n"); fclose(inFile); return 12; } alignedBuffer = (buffer+15) - (((size_t)(buffer+15)) & 0xF); // align on next 16 bytes boundaries // Fill input buffer DISPLAY("\rLoading %s... \n", inFileName); readSize = fread(alignedBuffer, 1, benchedSize, inFile); fclose(inFile); if(readSize != benchedSize) { DISPLAY("\nError: problem reading file '%s' !! \n", inFileName); free(buffer); return 13; } // Bench XXH32 { int interationNb; double fastestC = 100000000.; DISPLAY("\r%79s\r", ""); // Clean display line for (interationNb = 1; interationNb <= g_nbIterations; interationNb++) { int nbHashes = 0; int milliTime; DISPLAY("%1i-%-14.14s : %10i ->\r", interationNb, "XXH32", (int)benchedSize); // Hash loop milliTime = BMK_GetMilliStart(); while(BMK_GetMilliStart() == milliTime); milliTime = BMK_GetMilliStart(); while(BMK_GetMilliSpan(milliTime) < TIMELOOP) { int i; for (i=0; i<100; i++) { hashResult = XXH32(alignedBuffer, benchedSize, 0); nbHashes++; } } milliTime = BMK_GetMilliSpan(milliTime); if ((double)milliTime < fastestC*nbHashes) fastestC = (double)milliTime/nbHashes; DISPLAY("%1i-%-14.14s : %10i -> %7.1f MB/s\r", interationNb, "XXH32", (int)benchedSize, (double)benchedSize / fastestC / 1000.); } DISPLAY("%-16.16s : %10i -> %7.1f MB/s 0x%08X\n", "XXH32", (int)benchedSize, (double)benchedSize / fastestC / 1000., hashResult); totals += benchedSize; totalc += fastestC; } // Bench Unaligned XXH32 { int interationNb; double fastestC = 100000000.; DISPLAY("\r%79s\r", ""); // Clean display line for (interationNb = 1; (interationNb <= g_nbIterations) && ((benchedSize>1)); interationNb++) { int nbHashes = 0; int milliTime; DISPLAY("%1i-%-14.14s : %10i ->\r", interationNb, "(unaligned)", (int)benchedSize); // Hash loop milliTime = BMK_GetMilliStart(); while(BMK_GetMilliStart() == milliTime); milliTime = BMK_GetMilliStart(); while(BMK_GetMilliSpan(milliTime) < TIMELOOP) { int i; for (i=0; i<100; i++) { hashResult = XXH32(alignedBuffer+1, benchedSize-1, 0); nbHashes++; } } milliTime = BMK_GetMilliSpan(milliTime); if ((double)milliTime < fastestC*nbHashes) fastestC = (double)milliTime/nbHashes; DISPLAY("%1i-%-14.14s : %10i -> %7.1f MB/s\r", interationNb, "XXH32 (unaligned)", (int)(benchedSize-1), (double)(benchedSize-1) / fastestC / 1000.); } DISPLAY("%-16.16s : %10i -> %7.1f MB/s \n", "XXH32 (unaligned)", (int)benchedSize-1, (double)(benchedSize-1) / fastestC / 1000.); } // Bench XXH64 { int interationNb; double fastestC = 100000000.; unsigned long long h64 = 0; DISPLAY("\r%79s\r", ""); // Clean display line for (interationNb = 1; interationNb <= g_nbIterations; interationNb++) { int nbHashes = 0; int milliTime; DISPLAY("%1i-%-14.14s : %10i ->\r", interationNb, "XXH64", (int)benchedSize); // Hash loop milliTime = BMK_GetMilliStart(); while(BMK_GetMilliStart() == milliTime); milliTime = BMK_GetMilliStart(); while(BMK_GetMilliSpan(milliTime) < TIMELOOP) { int i; for (i=0; i<100; i++) { h64 = XXH64(alignedBuffer, benchedSize, 0); nbHashes++; } } milliTime = BMK_GetMilliSpan(milliTime); if ((double)milliTime < fastestC*nbHashes) fastestC = (double)milliTime/nbHashes; DISPLAY("%1i-%-14.14s : %10i -> %7.1f MB/s\r", interationNb, "XXH64", (int)benchedSize, (double)benchedSize / fastestC / 1000.); } DISPLAY("%-16.16s : %10i -> %7.1f MB/s 0x%08X%08X\n", "XXH64", (int)benchedSize, (double)benchedSize / fastestC / 1000., (U32)(h64>>32), (U32)(h64)); totals += benchedSize; totalc += fastestC; } free(buffer); } if (nbFiles > 1) printf("%-16.16s :%11llu -> %7.1f MB/s\n", " TOTAL", (long long unsigned int)totals, (double)totals/totalc/1000.); return 0; } static void BMK_checkResult(U32 r1, U32 r2) { static int nbTests = 1; if (r1==r2) DISPLAY("\rTest%3i : %08X == %08X ok ", nbTests, r1, r2); else { DISPLAY("\rERROR : Test%3i : %08X <> %08X !!!!! \n", nbTests, r1, r2); exit(1); } nbTests++; } static void BMK_checkResult64(U64 r1, U64 r2) { static int nbTests = 1; if (r1!=r2) { DISPLAY("\rERROR : Test%3i : 64-bits values non equals !!!!! \n", nbTests); DISPLAY("\r %08X%08X != %08X%08X \n", (U32)(r1>>32), (U32)r1, (U32)(r2<<32), (U32)r2); exit(1); } nbTests++; } static void BMK_testSequence64(void* sentence, int len, U64 seed, U64 Nresult) { U64 Dresult; XXH64_state_t state; int index; Dresult = XXH64(sentence, len, seed); BMK_checkResult64(Dresult, Nresult); XXH64_reset(&state, seed); XXH64_update(&state, sentence, len); Dresult = XXH64_digest(&state); BMK_checkResult64(Dresult, Nresult); XXH64_reset(&state, seed); for (index=0; index>24); prime *= prime; } BMK_testSequence(NULL, 0, 0, 0x02CC5D05); BMK_testSequence(NULL, 0, PRIME, 0x36B78AE7); BMK_testSequence(sanityBuffer, 1, 0, 0xB85CBEE5); BMK_testSequence(sanityBuffer, 1, PRIME, 0xD5845D64); BMK_testSequence(sanityBuffer, 14, 0, 0xE5AA0AB4); BMK_testSequence(sanityBuffer, 14, PRIME, 0x4481951D); BMK_testSequence(sanityBuffer, SANITY_BUFFER_SIZE, 0, 0x1F1AA412); BMK_testSequence(sanityBuffer, SANITY_BUFFER_SIZE, PRIME, 0x498EC8E2); BMK_testSequence64(NULL , 0, 0, 0xEF46DB3751D8E999ULL); BMK_testSequence64(NULL , 0, PRIME, 0xAC75FDA2929B17EFULL); BMK_testSequence64(sanityBuffer, 1, 0, 0x4FCE394CC88952D8ULL); BMK_testSequence64(sanityBuffer, 1, PRIME, 0x739840CB819FA723ULL); BMK_testSequence64(sanityBuffer, 14, 0, 0xCFFA8DB881BC3A3DULL); BMK_testSequence64(sanityBuffer, 14, PRIME, 0x5B9611585EFCC9CBULL); BMK_testSequence64(sanityBuffer, SANITY_BUFFER_SIZE, 0, 0x0EAB543384F878ADULL); BMK_testSequence64(sanityBuffer, SANITY_BUFFER_SIZE, PRIME, 0xCAA65939306F1E21ULL); DISPLAY("\r%79s\r", ""); // Clean display line DISPLAYLEVEL(2, "Sanity check -- all tests ok\n"); } static int BMK_hash(const char* fileName, U32 hashNb) { FILE* inFile; size_t const blockSize = 64 KB; size_t readSize; char* buffer; XXH64_state_t state; // Check file existence if (fileName == stdinName) { inFile = stdin; SET_BINARY_MODE(stdin); } else inFile = fopen( fileName, "rb" ); if (inFile==NULL) { DISPLAY( "Pb opening %s\n", fileName); return 11; } // Memory allocation & restrictions buffer = (char*)malloc(blockSize); if(!buffer) { DISPLAY("\nError: not enough memory!\n"); fclose(inFile); return 12; } // Init switch(hashNb) { case 0: XXH32_reset((XXH32_state_t*)&state, 0); break; case 1: XXH64_reset(&state, 0); break; default: DISPLAY("Error : bad hash algorithm ID\n"); fclose(inFile); free(buffer); return -1; } // Load file & update hash DISPLAY("\rLoading %s... \r", fileName); readSize = 1; while (readSize) { readSize = fread(buffer, 1, blockSize, inFile); switch(hashNb) { case 0: XXH32_update((XXH32_state_t*)&state, buffer, readSize); break; case 1: XXH64_update(&state, buffer, readSize); break; default: break; } } fclose(inFile); free(buffer); // display Hash switch(hashNb) { case 0: { U32 h32 = XXH32_digest((XXH32_state_t*)&state); DISPLAYRESULT("%08x %s \n", h32, fileName); break; } case 1: { U64 h64 = XXH64_digest(&state); DISPLAYRESULT("%08x%08x %s \n", (U32)(h64>>32), (U32)(h64), fileName); break; } default: break; } return 0; } //********************************************************* // Main //********************************************************* static int usage(const char* exename) { DISPLAY( WELCOME_MESSAGE ); DISPLAY( "Usage :\n"); DISPLAY( " %s [arg] [filename]\n", exename); DISPLAY( "When no filename provided, or - provided : use stdin as input\n"); DISPLAY( "Arguments :\n"); DISPLAY( " -H# : hash selection : 0=32bits, 1=64bits (default %i)\n", g_fn_selection); DISPLAY( " -b : benchmark mode \n"); DISPLAY( " -i# : number of iterations (benchmark mode; default %i)\n", g_nbIterations); DISPLAY( " -h : help (this text)\n"); return 0; } static int badusage(const char* exename) { DISPLAY("Wrong parameters\n"); usage(exename); return 1; } int main(int argc, char** argv) { int i, filenamesStart=0; const char* input_filename = (char*)stdinName; const char* exename = argv[0]; U32 benchmarkMode = 0; // xxh32sum default to 32 bits checksum if (strstr(exename, "xxh32sum")!=NULL) g_fn_selection=0; for(i=1; i 1) return badusage(exename); return BMK_hash(input_filename, g_fn_selection); }