From ba47d09c4b4d45aded7395b444f6010a6b45d02a Mon Sep 17 00:00:00 2001 From: Francisco Javier Trujillo Mata Date: Sat, 14 May 2022 18:23:32 +0200 Subject: [PATCH] [ORBIS] Update xxHash dependecy --- deps/xxHash/CHANGELOG | 19 + deps/xxHash/Doxyfile | 5 +- deps/xxHash/LICENSE | 24 +- deps/xxHash/Makefile | 88 +- deps/xxHash/README.md | 50 +- deps/xxHash/cli/xsum_arch.h | 20 +- deps/xxHash/cli/xsum_bench.c | 436 ++++ deps/xxHash/cli/xsum_bench.h | 51 + deps/xxHash/cli/xsum_config.h | 11 +- deps/xxHash/cli/xsum_os_specific.c | 18 +- deps/xxHash/cli/xsum_os_specific.h | 4 +- deps/xxHash/cli/xsum_output.c | 5 +- deps/xxHash/cli/xsum_output.h | 2 +- deps/xxHash/cli/xsum_sanity_check.c | 128 +- deps/xxHash/cli/xsum_sanity_check.h | 2 +- deps/xxHash/cli/xxhsum.1 | 69 +- deps/xxHash/cli/xxhsum.1.md | 16 +- deps/xxHash/cli/xxhsum.c | 676 ++----- deps/xxHash/cmake_unofficial/CMakeLists.txt | 12 +- deps/xxHash/doc/xxhash_spec.md | 2 +- deps/xxHash/libxxhash.pc.in | 2 +- deps/xxHash/tests/Makefile | 6 +- deps/xxHash/tests/bench/Makefile | 2 +- deps/xxHash/tests/bench/benchHash.c | 2 +- deps/xxHash/tests/bench/benchHash.h | 2 +- deps/xxHash/tests/bench/benchfn.c | 2 +- deps/xxHash/tests/bench/benchfn.h | 2 +- deps/xxHash/tests/bench/bhDisplay.c | 2 +- deps/xxHash/tests/bench/bhDisplay.h | 2 +- deps/xxHash/tests/bench/hashes.h | 2 +- deps/xxHash/tests/bench/main.c | 2 +- deps/xxHash/tests/bench/timefn.c | 2 +- deps/xxHash/tests/bench/timefn.h | 2 +- deps/xxHash/tests/collisions/Makefile | 2 +- deps/xxHash/tests/collisions/hashes.h | 2 +- deps/xxHash/tests/collisions/main.c | 4 +- deps/xxHash/tests/collisions/pool.c | 2 +- deps/xxHash/tests/collisions/pool.h | 2 +- deps/xxHash/tests/collisions/sort.cc | 2 +- deps/xxHash/tests/collisions/sort.hh | 2 +- deps/xxHash/xxh3.h | 2 +- deps/xxHash/xxh_x86dispatch.c | 8 +- deps/xxHash/xxh_x86dispatch.h | 3 +- deps/xxHash/xxhash.c | 2 +- deps/xxHash/xxhash.h | 2006 ++++++++++++------- 45 files changed, 2358 insertions(+), 1347 deletions(-) create mode 100644 deps/xxHash/cli/xsum_bench.c create mode 100644 deps/xxHash/cli/xsum_bench.h diff --git a/deps/xxHash/CHANGELOG b/deps/xxHash/CHANGELOG index 23870756bb..ff59d8bb1f 100644 --- a/deps/xxHash/CHANGELOG +++ b/deps/xxHash/CHANGELOG @@ -1,3 +1,22 @@ +v0.8.1 +- perf : much improved performance for XXH3 streaming variants, notably on gcc and msvc +- perf : improved XXH64 speed and latency on small inputs +- perf : small XXH32 speed and latency improvement on small inputs of random size +- perf : minor stack usage improvement for XXH32 and XXH64 +- api : new experimental variants XXH3_*_withSecretandSeed() +- api : update XXH3_generateSecret(), can no generate secret of any size (>= XXH3_SECRET_SIZE_MIN) +- cli : xxhsum can now generate and check XXH3 checksums, using command `-H3` +- build: can build xxhash without XXH3, with new build macro XXH_NO_XXH3 +- build: fix xxh_x86dispatch build with MSVC, by @apankrat +- build: XXH_INLINE_ALL can always be used safely, even after XXH_NAMESPACE or a previous XXH_INLINE_ALL +- build: improved PPC64LE vector support, by @mpe +- install: fix pkgconfig, by @ellert +- install: compatibility with Haiku, by @Begasus +- doc : code comments made compatible with doxygen, by @easyaspi314 +- misc : XXH_ACCEPT_NULL_INPUT_POINTER is no longer necessary, all functions can accept NULL input pointers, as long as size == 0 +- misc : complete refactor of CI tests on Github Actions, offering much larger coverage, by @t-mat +- misc : xxhsum code base split into multiple specialized units, within directory cli/, by @easyaspi314 + v0.8.0 - api : stabilize XXH3 - cli : xxhsum can parse BSD-style --check lines, by @WayneD diff --git a/deps/xxHash/Doxyfile b/deps/xxHash/Doxyfile index 634e1e1a38..f1f8512a22 100644 --- a/deps/xxHash/Doxyfile +++ b/deps/xxHash/Doxyfile @@ -2,7 +2,7 @@ DOXYFILE_ENCODING = UTF-8 PROJECT_NAME = "xxHash" -PROJECT_NUMBER = "0.8.0" +PROJECT_NUMBER = "0.8.1" PROJECT_BRIEF = "Extremely fast non-cryptographic hash function" OUTPUT_DIRECTORY = doxygen OUTPUT_LANGUAGE = English @@ -42,6 +42,9 @@ PREDEFINED = "XXH_DOXYGEN=" \ "XSUM_API=" \ "XXH_STATIC_LINKING_ONLY" \ "XXH_IMPLEMENTATION" \ + "XXH_PUREF=[[gnu::pure]]" \ + "XXH_CONSTF=[[gnu::const]]" \ + "XXH_MALLOCF=[[gnu::malloc]]" \ "XXH_ALIGN(N)=alignas(N)" \ "XXH_ALIGN_MEMBER(align,type)=alignas(align) type" diff --git a/deps/xxHash/LICENSE b/deps/xxHash/LICENSE index fa20595dc6..e4c5da7234 100644 --- a/deps/xxHash/LICENSE +++ b/deps/xxHash/LICENSE @@ -1,5 +1,5 @@ xxHash Library -Copyright (c) 2012-2020 Yann Collet +Copyright (c) 2012-2021 Yann Collet All rights reserved. BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php) @@ -24,25 +24,3 @@ LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - ----------------------------------------------------- - -xxhsum command line interface -Copyright (c) 2013-2020 Yann Collet -All rights reserved. - -GPL v2 License - -This program is free software; you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation; either version 2 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License along -with this program; if not, write to the Free Software Foundation, Inc., -51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. diff --git a/deps/xxHash/Makefile b/deps/xxHash/Makefile index da6e08187c..549bababd1 100644 --- a/deps/xxHash/Makefile +++ b/deps/xxHash/Makefile @@ -1,6 +1,6 @@ # ################################################################ # xxHash Makefile -# Copyright (C) 2012-2020 Yann Collet +# Copyright (C) 2012-2021 Yann Collet # # GPL v2 License # @@ -75,13 +75,15 @@ XXHSUM_SRC_DIR = cli XXHSUM_SPLIT_SRCS = $(XXHSUM_SRC_DIR)/xxhsum.c \ $(XXHSUM_SRC_DIR)/xsum_os_specific.c \ $(XXHSUM_SRC_DIR)/xsum_output.c \ - $(XXHSUM_SRC_DIR)/xsum_sanity_check.c + $(XXHSUM_SRC_DIR)/xsum_sanity_check.c \ + $(XXHSUM_SRC_DIR)/xsum_bench.c XXHSUM_SPLIT_OBJS = $(XXHSUM_SPLIT_SRCS:.c=.o) XXHSUM_HEADERS = $(XXHSUM_SRC_DIR)/xsum_config.h \ $(XXHSUM_SRC_DIR)/xsum_arch.h \ $(XXHSUM_SRC_DIR)/xsum_os_specific.h \ $(XXHSUM_SRC_DIR)/xsum_output.h \ - $(XXHSUM_SRC_DIR)/xsum_sanity_check.h + $(XXHSUM_SRC_DIR)/xsum_sanity_check.h \ + $(XXHSUM_SRC_DIR)/xsum_bench.h ## generate CLI and libraries in release mode (default for `make`) .PHONY: default @@ -204,6 +206,8 @@ check: xxhsum ## basic tests for xxhsum CLI, set RUN_ENV for emulated environm $(RUN_ENV) ./xxhsum$(EXT) -H0 xxhash.c # 128-bit $(RUN_ENV) ./xxhsum$(EXT) -H2 xxhash.c + # XXH3 (enforce BSD style) + $(RUN_ENV) ./xxhsum$(EXT) -H3 xxhash.c | grep "XXH3" # request incorrect variant $(RUN_ENV) ./xxhsum$(EXT) -H9 xxhash.c ; test $$? -eq 1 @printf "\n ....... checks completed successfully ....... \n" @@ -250,12 +254,18 @@ test-xxhsum-c: xxhsum ./xxhsum -H2 --tag $(TEST_FILES) > .test.xxh128_tag ./xxhsum -H2 --little-endian $(TEST_FILES) > .test.le_xxh128 ./xxhsum -H2 --tag --little-endian $(TEST_FILES) > .test.le_xxh128_tag + ./xxhsum -H3 $(TEST_FILES) > .test.xxh3 + ./xxhsum -H3 --tag $(TEST_FILES) > .test.xxh3_tag + ./xxhsum -H3 --little-endian $(TEST_FILES) > .test.le_xxh3 + ./xxhsum -H3 --tag --little-endian $(TEST_FILES) > .test.le_xxh3_tag ./xxhsum -c .test.xxh* ./xxhsum -c --little-endian .test.le_xxh* ./xxhsum -c .test.*_tag # read list of files from stdin - ./xxhsum -c < .test.xxh64 ./xxhsum -c < .test.xxh32 + ./xxhsum -c < .test.xxh64 + ./xxhsum -c < .test.xxh128 + ./xxhsum -c < .test.xxh3 cat .test.xxh* | ./xxhsum -c - # check variant with '*' marker as second separator $(SED) 's/ / \*/' .test.xxh32 | ./xxhsum -c @@ -264,12 +274,15 @@ test-xxhsum-c: xxhsum ./xxhsum --tag -H0 xxhsum* | $(GREP) XXH32 ./xxhsum --tag -H1 xxhsum* | $(GREP) XXH64 ./xxhsum --tag -H2 xxhsum* | $(GREP) XXH128 + ./xxhsum --tag -H3 xxhsum* | $(GREP) XXH3 + ./xxhsum -H3 xxhsum* | $(GREP) XXH3 # --tag is implicit for H3 ./xxhsum --tag -H32 xxhsum* | $(GREP) XXH32 ./xxhsum --tag -H64 xxhsum* | $(GREP) XXH64 ./xxhsum --tag -H128 xxhsum* | $(GREP) XXH128 ./xxhsum --tag -H0 --little-endian xxhsum* | $(GREP) XXH32_LE ./xxhsum --tag -H1 --little-endian xxhsum* | $(GREP) XXH64_LE ./xxhsum --tag -H2 --little-endian xxhsum* | $(GREP) XXH128_LE + ./xxhsum -H3 --little-endian xxhsum* | $(GREP) XXH3_LE ./xxhsum --tag -H32 --little-endian xxhsum* | $(GREP) XXH32_LE ./xxhsum --tag -H64 --little-endian xxhsum* | $(GREP) XXH64_LE ./xxhsum --tag -H128 --little-endian xxhsum* | $(GREP) XXH128_LE @@ -289,6 +302,10 @@ test-xxhsum-c: xxhsum echo "00000000 test-expects-file-not-found" | ./xxhsum -c -; test $$? -eq 1 @$(RM) .test.* +.PHONY: test-filename-escape +test-filename-escape: + $(MAKE) -C tests test_filename_escape + .PHONY: armtest armtest: clean @echo ---- test ARM compilation ---- @@ -319,14 +336,35 @@ c90test: xxhash.c $(RM) xxhash.o endif +.PHONY: noxxh3test noxxh3test: CPPFLAGS += -DXXH_NO_XXH3 noxxh3test: CFLAGS += -Werror -pedantic -Wno-long-long # XXH64 requires long long support +noxxh3test: OFILE = xxh_noxxh3.o noxxh3test: xxhash.c @echo ---- test compilation without XXH3 ---- - $(RM) xxhash.o - $(CC) $(FLAGS) $^ -c - $(NM) xxhash.o | $(GREP) XXH3_ ; test $$? -eq 1 - $(RM) xxhash.o + $(CC) $(FLAGS) -c $^ -o $(OFILE) + $(NM) $(OFILE) | $(GREP) XXH3_ ; test $$? -eq 1 + $(RM) $(OFILE) + +.PHONY: nostreamtest +nostreamtest: CPPFLAGS += -DXXH_NO_STREAM +nostreamtest: CFLAGS += -Werror -pedantic -Wno-long-long # XXH64 requires long long support +nostreamtest: OFILE = xxh_nostream.o +nostreamtest: xxhash.c + @echo ---- test compilation without streaming ---- + $(CC) $(FLAGS) -c $^ -o $(OFILE) + $(NM) $(OFILE) | $(GREP) update ; test $$? -eq 1 + $(RM) $(OFILE) + +.PHONY: nostdlibtest +nostdlibtest: CPPFLAGS += -DXXH_NO_STDLIB +nostdlibtest: CFLAGS += -Werror -pedantic -Wno-long-long # XXH64 requires long long support +nostdlibtest: OFILE = xxh_nostdlib.o +nostdlibtest: xxhash.c + @echo ---- test compilation without \ ---- + $(CC) $(FLAGS) -c $^ -o $(OFILE) + $(NM) $(OFILE) | $(GREP) "U _free\|U free" ; test $$? -eq 1 + $(RM) $(OFILE) .PHONY: usan usan: CC=clang @@ -374,7 +412,7 @@ preview-man: man .PHONY: test test: DEBUGFLAGS += -DXXH_DEBUGLEVEL=1 -test: all namespaceTest check test-xxhsum-c c90test test-tools +test: all namespaceTest check test-xxhsum-c c90test test-tools noxxh3test nostdlibtest .PHONY: test-inline test-inline: @@ -444,6 +482,7 @@ endif INSTALL_PROGRAM ?= $(INSTALL) INSTALL_DATA ?= $(INSTALL) -m 644 +INSTALL_DIR ?= $(INSTALL) -d -m 755 # Escape special symbols by putting each character into its separate class @@ -480,11 +519,14 @@ libxxhash.pc: libxxhash.pc.in $< > $@ -.PHONY: install -install: lib libxxhash.pc xxhsum ## install libraries, CLI, links and man page - @echo Installing libxxhash - $(Q)$(INSTALL) -d -m 755 $(DESTDIR)$(LIBDIR) +install_libxxhash.a: libxxhash.a + @echo Installing libxxhash.a + $(Q)$(INSTALL_DIR) $(DESTDIR)$(LIBDIR) $(Q)$(INSTALL_DATA) libxxhash.a $(DESTDIR)$(LIBDIR) + +install_libxxhash: libxxhash + @echo Installing libxxhash + $(Q)$(INSTALL_DIR) $(DESTDIR)$(LIBDIR) $(Q)$(INSTALL_PROGRAM) $(LIBXXH) $(DESTDIR)$(LIBDIR) $(Q)ln -sf $(LIBXXH) $(DESTDIR)$(LIBDIR)/libxxhash.$(SHARED_EXT_MAJOR) $(Q)ln -sf $(LIBXXH) $(DESTDIR)$(LIBDIR)/libxxhash.$(SHARED_EXT) @@ -494,20 +536,30 @@ install: lib libxxhash.pc xxhsum ## install libraries, CLI, links and man page ifeq ($(DISPATCH),1) $(Q)$(INSTALL_DATA) xxh_x86dispatch.h $(DESTDIR)$(INCLUDEDIR) endif + +install_libxxhash.pc: libxxhash.pc @echo Installing pkgconfig - $(Q)$(INSTALL) -d -m 755 $(DESTDIR)$(PKGCONFIGDIR)/ + $(Q)$(INSTALL_DIR) $(DESTDIR)$(PKGCONFIGDIR)/ $(Q)$(INSTALL_DATA) libxxhash.pc $(DESTDIR)$(PKGCONFIGDIR)/ + +install_xxhsum: xxhsum @echo Installing xxhsum - $(Q)$(INSTALL) -d -m 755 $(DESTDIR)$(BINDIR)/ $(DESTDIR)$(MANDIR)/ + $(Q)$(INSTALL_DIR) $(DESTDIR)$(BINDIR)/ $(Q)$(INSTALL_PROGRAM) xxhsum $(DESTDIR)$(BINDIR)/xxhsum $(Q)ln -sf xxhsum $(DESTDIR)$(BINDIR)/xxh32sum $(Q)ln -sf xxhsum $(DESTDIR)$(BINDIR)/xxh64sum $(Q)ln -sf xxhsum $(DESTDIR)$(BINDIR)/xxh128sum + +install_man: @echo Installing man pages + $(Q)$(INSTALL_DIR) $(DESTDIR)$(MANDIR)/ $(Q)$(INSTALL_DATA) $(MAN) $(DESTDIR)$(MANDIR)/xxhsum.1 - $(Q)ln -sf $(MAN) $(DESTDIR)$(MANDIR)/xxh32sum.1 - $(Q)ln -sf $(MAN) $(DESTDIR)$(MANDIR)/xxh64sum.1 - $(Q)ln -sf $(MAN) $(DESTDIR)$(MANDIR)/xxh128sum.1 + $(Q)ln -sf xxhsum.1 $(DESTDIR)$(MANDIR)/xxh32sum.1 + $(Q)ln -sf xxhsum.1 $(DESTDIR)$(MANDIR)/xxh64sum.1 + $(Q)ln -sf xxhsum.1 $(DESTDIR)$(MANDIR)/xxh128sum.1 + +.PHONY: install +install: install_libxxhash.a install_libxxhash install_libxxhash.pc install_xxhsum install_man ## install libraries, CLI, links and man page @echo xxhash installation completed .PHONY: uninstall diff --git a/deps/xxHash/README.md b/deps/xxHash/README.md index 18704f5300..e8b57adc39 100644 --- a/deps/xxHash/README.md +++ b/deps/xxHash/README.md @@ -9,6 +9,7 @@ Code is highly portable, and hashes are identical across all platforms (little / |Branch |Status | |------------|---------| +|release | [![Build Status](https://github.com/Cyan4973/xxHash/actions/workflows/ci.yml/badge.svg?branch=release)](https://github.com/Cyan4973/xxHash/actions?query=branch%3Arelease+) | |dev | [![Build Status](https://github.com/Cyan4973/xxHash/actions/workflows/ci.yml/badge.svg?branch=dev)](https://github.com/Cyan4973/xxHash/actions?query=branch%3Adev+) | @@ -110,35 +111,42 @@ The following macros can be set at compilation time to modify libxxhash's behavi It is (slightly) detrimental on platform with good unaligned memory access performance (same instruction for both aligned and unaligned accesses). This option is automatically disabled on `x86`, `x64` and `aarch64`, and enabled on all other platforms. - `XXH_VECTOR` : manually select a vector instruction set (default: auto-selected at compilation time). Available instruction sets are `XXH_SCALAR`, `XXH_SSE2`, `XXH_AVX2`, `XXH_AVX512`, `XXH_NEON` and `XXH_VSX`. Compiler may require additional flags to ensure proper support (for example, `gcc` on linux will require `-mavx2` for AVX2, and `-mavx512f` for AVX512). -- `XXH_NO_PREFETCH` : disable prefetching. XXH3 only. -- `XXH_PREFETCH_DIST` : select prefecting distance. XXH3 only. +- `XXH_NO_PREFETCH` : disable prefetching. Some platforms or situations may perform better without prefetching. XXH3 only. +- `XXH_PREFETCH_DIST` : select prefetching distance. For close-to-metal adaptation to specific hardware platforms. XXH3 only. +- `XXH_NO_STREAM`: Disables the streaming API, limiting it to single shot variants only. +- `XXH_SIZE_OPT`: `0`: default, optimize for speed + `1`: default for `-Os` and `-Oz`: disables some speed hacks for size optimization + `2`: makes code as small as possible, performance may cry - `XXH_NO_INLINE_HINTS`: By default, xxHash uses `__attribute__((always_inline))` and `__forceinline` to improve performance at the cost of code size. Defining this macro to 1 will mark all internal functions as `static`, allowing the compiler to decide whether to inline a function or not. This is very useful when optimizing for smallest binary size, and is automatically defined when compiling with `-O0`, `-Os`, `-Oz`, or `-fno-inline` on GCC and Clang. This may also increase performance depending on compiler and architecture. -- `XXH_REROLL`: Reduces the size of the generated code by not unrolling some loops. - Impact on performance may vary, depending on platform and algorithm. -- `XXH_ACCEPT_NULL_INPUT_POINTER`: if set to `1`, when input is a `NULL` pointer, - xxHash'd result is the same as a zero-length input - (instead of a dereference segfault). - Adds one branch at the beginning of each hash. -- `XXH_STATIC_LINKING_ONLY`: gives access to the state declaration for static allocation. +- `XXH32_ENDJMP`: Switch multi-branch finalization stage of XXH32 by a single jump. + This is generally undesirable for performance, especially when hashing inputs of random sizes. + But depending on exact architecture and compiler, a jump might provide slightly better performance on small inputs. Disabled by default. +- `XXH_NO_STDLIB`: Disable invocation of `` functions, notably `malloc()` and `free()`. + `libxxhash`'s `XXH*_createState()` will always fail and return `NULL`. + But one-shot hashing (like `XXH32()`) or streaming using statically allocated states + still work as expected. + This build flag is useful for embedded environments without dynamic allocation. +- `XXH_STATIC_LINKING_ONLY`: gives access to internal state declaration, required for static allocation. Incompatible with dynamic linking, due to risks of ABI changes. - `XXH_NO_XXH3` : removes symbols related to `XXH3` (both 64 & 128 bits) from generated binary. Useful to reduce binary size, notably for applications which do not use `XXH3`. - `XXH_NO_LONG_LONG`: removes compilation of algorithms relying on 64-bit types (XXH3 and XXH64). Only XXH32 will be compiled. Useful for targets (architectures and compilers) without 64-bit support. - `XXH_IMPORT`: MSVC specific: should only be defined for dynamic linking, as it prevents linkage errors. -- `XXH_CPU_LITTLE_ENDIAN`: By default, endianess is determined by a runtime test resolved at compile time. +- `XXH_CPU_LITTLE_ENDIAN`: By default, endianness is determined by a runtime test resolved at compile time. If, for some reason, the compiler cannot simplify the runtime test, it can cost performance. It's possible to skip auto-detection and simply state that the architecture is little-endian by setting this macro to 1. Setting it to 0 states big-endian. +- `XXH_DEBUGLEVEL` : When set to any value >= 1, enables `assert()` statements. + This (slightly) slows down execution, but may help finding bugs during debugging sessions. -For the Command Line Interface `xxhsum`, the following environment variables can also be set : +When compiling the Command Line Interface `xxhsum` with `make`, the following environment variables can also be set : - `DISPATCH=1` : use `xxh_x86dispatch.c`, to automatically select between `scalar`, `sse2`, `avx2` or `avx512` instruction set at runtime, depending on local host. This option is only valid for `x86`/`x64` systems. - ### Building xxHash - Using vcpkg You can download and install xxHash using the [vcpkg](https://github.com/Microsoft/vcpkg) dependency manager: @@ -151,6 +159,24 @@ You can download and install xxHash using the [vcpkg](https://github.com/Microso The xxHash port in vcpkg is kept up to date by Microsoft team members and community contributors. If the version is out of date, please [create an issue or pull request](https://github.com/Microsoft/vcpkg) on the vcpkg repository. +### Building and Using xxHash - tipi.build + +You can work on xxHash and depend on it in your [tipi.build](https://tipi.build) projects by adding the following entry to your `.tipi/deps`: + +```json +{ + "Cyan4973/xxHash": { "@": "v0.8.1" } +} +``` + +An example of such usage can be found in the `/cli` folder of this project which, if built as root project will depend on the release `v0.8.1` of xxHash + + +To contribute to xxHash itself use tipi.build on this repository (change the target name appropriately to `linux` or `macos` or `windows`): + +```bash +tipi . -t --test all +``` ### Example diff --git a/deps/xxHash/cli/xsum_arch.h b/deps/xxHash/cli/xsum_arch.h index cc39297972..17d332cb68 100644 --- a/deps/xxHash/cli/xsum_arch.h +++ b/deps/xxHash/cli/xsum_arch.h @@ -1,6 +1,6 @@ /* * xxhsum - Command line interface for xxhash algorithms - * Copyright (C) 2013-2020 Yann Collet + * Copyright (C) 2013-2021 Yann Collet * * GPL v2 License * @@ -79,7 +79,7 @@ #endif /* makes the next part easier */ -#if defined(__x86_64__) || defined(_M_AMD64) || defined(_M_X64) +#if (defined(__x86_64__) || defined(_M_AMD64) || defined(_M_X64)) && !defined(_M_ARM64EC) # define XSUM_ARCH_X64 1 # define XSUM_ARCH_X86 "x86_64" #elif defined(__i386__) || defined(_M_IX86) || defined(_M_IX86_FP) @@ -102,11 +102,17 @@ # else # define XSUM_ARCH XSUM_ARCH_X86 # endif -#elif defined(__aarch64__) || defined(__arm64__) || defined(_M_ARM64) +#elif defined(__aarch64__) || defined(__arm64__) || defined(_M_ARM64) || defined(_M_ARM64EC) # define XSUM_ARCH "aarch64 + NEON" #elif defined(__arm__) || defined(__thumb__) || defined(__thumb2__) || defined(_M_ARM) /* ARM has a lot of different features that can change xxHash significantly. */ -# if defined(__thumb2__) || (defined(__thumb__) && (__thumb__ == 2 || __ARM_ARCH >= 7)) +# ifdef __ARM_ARCH +# define XSUM_ARCH_ARM_VER XSUM_EXPAND_AND_QUOTE(__ARM_ARCH) +# else +# define XSUM_ARCH_ARM_VER XSUM_EXPAND_AND_QUOTE(_M_ARM) +# endif +# if defined(_M_ARM) /* windows arm is always thumb-2 */ \ + || defined(__thumb2__) || (defined(__thumb__) && (__thumb__ == 2 || __ARM_ARCH >= 7)) # define XSUM_ARCH_THUMB " Thumb-2" # elif defined(__thumb__) # define XSUM_ARCH_THUMB " Thumb-1" @@ -114,17 +120,17 @@ # define XSUM_ARCH_THUMB "" # endif /* ARMv7 has unaligned by default */ -# if defined(__ARM_FEATURE_UNALIGNED) || __ARM_ARCH >= 7 || defined(_M_ARMV7VE) +# if defined(__ARM_FEATURE_UNALIGNED) || __ARM_ARCH >= 7 || defined(_M_ARM) # define XSUM_ARCH_UNALIGNED " + unaligned" # else # define XSUM_ARCH_UNALIGNED "" # endif -# if defined(__ARM_NEON) || defined(__ARM_NEON__) +# if defined(__ARM_NEON) || defined(__ARM_NEON__) || defined(_M_ARM) # define XSUM_ARCH_NEON " + NEON" # else # define XSUM_ARCH_NEON "" # endif -# define XSUM_ARCH "ARMv" XSUM_EXPAND_AND_QUOTE(__ARM_ARCH) XSUM_ARCH_THUMB XSUM_ARCH_NEON XSUM_ARCH_UNALIGNED +# define XSUM_ARCH "ARMv" XSUM_ARCH_ARM_VER XSUM_ARCH_THUMB XSUM_ARCH_NEON XSUM_ARCH_UNALIGNED #elif defined(__powerpc64__) || defined(__ppc64__) || defined(__PPC64__) # if defined(__GNUC__) && defined(__POWER9_VECTOR__) # define XSUM_ARCH "ppc64 + POWER9 vector" diff --git a/deps/xxHash/cli/xsum_bench.c b/deps/xxHash/cli/xsum_bench.c new file mode 100644 index 0000000000..2285bb0931 --- /dev/null +++ b/deps/xxHash/cli/xsum_bench.c @@ -0,0 +1,436 @@ +/* + * xsum_bench - Benchmark functions for xxhsum + * Copyright (C) 2013-2021 Yann Collet + * + * GPL v2 License + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + * + * You can contact the author at: + * - xxHash homepage: https://www.xxhash.com + * - xxHash source repository: https://github.com/Cyan4973/xxHash + */ + +#include "xsum_output.h" /* XSUM_logLevel */ +#include "xsum_bench.h" +#include "xsum_sanity_check.h" /* XSUM_fillTestBuffer */ +#include "xsum_os_specific.h" /* XSUM_getFileSize */ +#ifndef XXH_STATIC_LINKING_ONLY +# define XXH_STATIC_LINKING_ONLY +#endif +#include "../xxhash.h" +#ifdef XXHSUM_DISPATCH +# include "../xxh_x86dispatch.h" /* activate _dispatch() redirectors */ +#endif + +#include /* malloc, free */ +#include +#include /* strlen, memcpy */ +#include /* clock_t, clock, CLOCKS_PER_SEC */ +#include /* errno */ + +#define TIMELOOP_S 1 +#define TIMELOOP (TIMELOOP_S * CLOCKS_PER_SEC) /* target timing per iteration */ +#define TIMELOOP_MIN (TIMELOOP / 2) /* minimum timing to validate a result */ + +#define MAX_MEM (2 GB - 64 MB) + +static clock_t XSUM_clockSpan( clock_t start ) +{ + return clock() - start; /* works even if overflow; Typical max span ~ 30 mn */ +} + +static size_t XSUM_findMaxMem(XSUM_U64 requiredMem) +{ + size_t const step = 64 MB; + void* testmem = NULL; + + requiredMem = (((requiredMem >> 26) + 1) << 26); + requiredMem += 2*step; + if (requiredMem > MAX_MEM) requiredMem = MAX_MEM; + + while (!testmem) { + if (requiredMem > step) requiredMem -= step; + else requiredMem >>= 1; + testmem = malloc ((size_t)requiredMem); + } + free (testmem); + + /* keep some space available */ + if (requiredMem > step) requiredMem -= step; + else requiredMem >>= 1; + + return (size_t)requiredMem; +} + +/* + * A secret buffer used for benchmarking XXH3's withSecret variants. + * + * In order for the bench to be realistic, the secret buffer would need to be + * pre-generated. + * + * Adding a pointer to the parameter list would be messy. + */ +static XSUM_U8 g_benchSecretBuf[XXH3_SECRET_SIZE_MIN]; + +/* + * Wrappers for the benchmark. + * + * If you would like to add other hashes to the bench, create a wrapper and add + * it to the g_hashesToBench table. It will automatically be added. + */ +typedef XSUM_U32 (*hashFunction)(const void* buffer, size_t bufferSize, XSUM_U32 seed); + +static XSUM_U32 localXXH32(const void* buffer, size_t bufferSize, XSUM_U32 seed) +{ + return XXH32(buffer, bufferSize, seed); +} +static XSUM_U32 localXXH32_stream(const void* buffer, size_t bufferSize, XSUM_U32 seed) +{ + XXH32_state_t state; + (void)seed; + XXH32_reset(&state, seed); + XXH32_update(&state, buffer, bufferSize); + return (XSUM_U32)XXH32_digest(&state); +} +static XSUM_U32 localXXH64(const void* buffer, size_t bufferSize, XSUM_U32 seed) +{ + return (XSUM_U32)XXH64(buffer, bufferSize, seed); +} +static XSUM_U32 localXXH64_stream(const void* buffer, size_t bufferSize, XSUM_U32 seed) +{ + XXH64_state_t state; + (void)seed; + XXH64_reset(&state, seed); + XXH64_update(&state, buffer, bufferSize); + return (XSUM_U32)XXH64_digest(&state); +} +static XSUM_U32 localXXH3_64b(const void* buffer, size_t bufferSize, XSUM_U32 seed) +{ + (void)seed; + return (XSUM_U32)XXH3_64bits(buffer, bufferSize); +} +static XSUM_U32 localXXH3_64b_seeded(const void* buffer, size_t bufferSize, XSUM_U32 seed) +{ + return (XSUM_U32)XXH3_64bits_withSeed(buffer, bufferSize, seed); +} +static XSUM_U32 localXXH3_64b_secret(const void* buffer, size_t bufferSize, XSUM_U32 seed) +{ + (void)seed; + return (XSUM_U32)XXH3_64bits_withSecret(buffer, bufferSize, g_benchSecretBuf, sizeof(g_benchSecretBuf)); +} +static XSUM_U32 localXXH3_128b(const void* buffer, size_t bufferSize, XSUM_U32 seed) +{ + (void)seed; + return (XSUM_U32)(XXH3_128bits(buffer, bufferSize).low64); +} +static XSUM_U32 localXXH3_128b_seeded(const void* buffer, size_t bufferSize, XSUM_U32 seed) +{ + return (XSUM_U32)(XXH3_128bits_withSeed(buffer, bufferSize, seed).low64); +} +static XSUM_U32 localXXH3_128b_secret(const void* buffer, size_t bufferSize, XSUM_U32 seed) +{ + (void)seed; + return (XSUM_U32)(XXH3_128bits_withSecret(buffer, bufferSize, g_benchSecretBuf, sizeof(g_benchSecretBuf)).low64); +} +static XSUM_U32 localXXH3_stream(const void* buffer, size_t bufferSize, XSUM_U32 seed) +{ + XXH3_state_t state; + (void)seed; + XXH3_64bits_reset(&state); + XXH3_64bits_update(&state, buffer, bufferSize); + return (XSUM_U32)XXH3_64bits_digest(&state); +} +static XSUM_U32 localXXH3_stream_seeded(const void* buffer, size_t bufferSize, XSUM_U32 seed) +{ + XXH3_state_t state; + XXH3_INITSTATE(&state); + XXH3_64bits_reset_withSeed(&state, (XXH64_hash_t)seed); + XXH3_64bits_update(&state, buffer, bufferSize); + return (XSUM_U32)XXH3_64bits_digest(&state); +} +static XSUM_U32 localXXH128_stream(const void* buffer, size_t bufferSize, XSUM_U32 seed) +{ + XXH3_state_t state; + (void)seed; + XXH3_128bits_reset(&state); + XXH3_128bits_update(&state, buffer, bufferSize); + return (XSUM_U32)(XXH3_128bits_digest(&state).low64); +} +static XSUM_U32 localXXH128_stream_seeded(const void* buffer, size_t bufferSize, XSUM_U32 seed) +{ + XXH3_state_t state; + XXH3_INITSTATE(&state); + XXH3_128bits_reset_withSeed(&state, (XXH64_hash_t)seed); + XXH3_128bits_update(&state, buffer, bufferSize); + return (XSUM_U32)(XXH3_128bits_digest(&state).low64); +} + + +typedef struct { + const char* name; + hashFunction func; +} hashInfo; + +static const hashInfo g_hashesToBench[] = { + { "XXH32", &localXXH32 }, + { "XXH64", &localXXH64 }, + { "XXH3_64b", &localXXH3_64b }, + { "XXH3_64b w/seed", &localXXH3_64b_seeded }, + { "XXH3_64b w/secret", &localXXH3_64b_secret }, + { "XXH128", &localXXH3_128b }, + { "XXH128 w/seed", &localXXH3_128b_seeded }, + { "XXH128 w/secret", &localXXH3_128b_secret }, + { "XXH32_stream", &localXXH32_stream }, + { "XXH64_stream", &localXXH64_stream }, + { "XXH3_stream", &localXXH3_stream }, + { "XXH3_stream w/seed",&localXXH3_stream_seeded }, + { "XXH128_stream", &localXXH128_stream }, + { "XXH128_stream w/seed",&localXXH128_stream_seeded }, +}; +#define NB_HASHFUNC (sizeof(g_hashesToBench) / sizeof(*g_hashesToBench)) + +#define NB_TESTFUNC (1 + 2 * NB_HASHFUNC) +int const g_nbTestFunctions = NB_TESTFUNC; +char g_testIDs[NB_TESTFUNC] = { 0 }; +const char k_testIDs_default[NB_TESTFUNC] = { 0, + 1 /*XXH32*/, 0, + 1 /*XXH64*/, 0, + 1 /*XXH3*/, 0, 0, 0, 0, 0, + 1 /*XXH128*/ }; + +int g_nbIterations = NBLOOPS_DEFAULT; +#define HASHNAME_MAX 29 +static void XSUM_benchHash(hashFunction h, const char* hName, int testID, + const void* buffer, size_t bufferSize) +{ + XSUM_U32 nbh_perIteration = (XSUM_U32)((300 MB) / (bufferSize+1)) + 1; /* first iteration conservatively aims for 300 MB/s */ + int iterationNb, nbIterations = g_nbIterations + !g_nbIterations /* min 1 */; + double fastestH = 100000000.; + assert(HASHNAME_MAX > 2); + XSUM_logVerbose(2, "\r%80s\r", ""); /* Clean display line */ + + for (iterationNb = 1; iterationNb <= nbIterations; iterationNb++) { + XSUM_U32 r=0; + clock_t cStart; + + XSUM_logVerbose(2, "%2i-%-*.*s : %10u ->\r", + iterationNb, + HASHNAME_MAX, HASHNAME_MAX, hName, + (unsigned)bufferSize); + cStart = clock(); + while (clock() == cStart); /* starts clock() at its exact beginning */ + cStart = clock(); + + { XSUM_U32 u; + for (u=0; u (double)(4000U<<20)) nbh_perSecond = (double)(4000U<<20); /* avoid overflow */ + nbh_perIteration = (XSUM_U32)nbh_perSecond; + } + /* g_nbIterations==0 => quick evaluation, no claim of accuracy */ + if (g_nbIterations>0) { + iterationNb--; /* new round for a more accurate speed evaluation */ + continue; + } + } + if (ticksPerHash < fastestH) fastestH = ticksPerHash; + if (fastestH>0.) { /* avoid div by zero */ + XSUM_logVerbose(2, "%2i-%-*.*s : %10u -> %8.0f it/s (%7.1f MB/s) \r", + iterationNb, + HASHNAME_MAX, HASHNAME_MAX, hName, + (unsigned)bufferSize, + (double)1 / fastestH, + ((double)bufferSize / (1 MB)) / fastestH); + } } + { double nbh_perSecond = (1 / fastestH) + 1; + if (nbh_perSecond > (double)(4000U<<20)) nbh_perSecond = (double)(4000U<<20); /* avoid overflow */ + nbh_perIteration = (XSUM_U32)nbh_perSecond; + } + } + XSUM_logVerbose(1, "%2i#%-*.*s : %10u -> %8.0f it/s (%7.1f MB/s) \n", + testID, + HASHNAME_MAX, HASHNAME_MAX, hName, + (unsigned)bufferSize, + (double)1 / fastestH, + ((double)bufferSize / (1 MB)) / fastestH); + if (XSUM_logLevel<1) + XSUM_logVerbose(0, "%u, ", (unsigned)((double)1 / fastestH)); +} + + +/* + * Allocates a string containing s1 and s2 concatenated. Acts like strdup. + * The result must be freed. + */ +static char* XSUM_strcatDup(const char* s1, const char* s2) +{ + assert(s1 != NULL); + assert(s2 != NULL); + { size_t len1 = strlen(s1); + size_t len2 = strlen(s2); + char* buf = (char*)malloc(len1 + len2 + 1); + if (buf != NULL) { + /* strcpy(buf, s1) */ + memcpy(buf, s1, len1); + /* strcat(buf, s2) */ + memcpy(buf + len1, s2, len2 + 1); + } + return buf; + } +} + + +/*! + * XSUM_benchMem(): + * buffer: Must be 16-byte aligned. + * The real allocated size of buffer is supposed to be >= (bufferSize+3). + * returns: 0 on success, 1 if error (invalid mode selected) + */ +static void XSUM_benchMem(const void* buffer, size_t bufferSize) +{ + assert((((size_t)buffer) & 15) == 0); /* ensure alignment */ + XSUM_fillTestBuffer(g_benchSecretBuf, sizeof(g_benchSecretBuf)); + { int i; + for (i = 1; i < (int)NB_TESTFUNC; i++) { + int const hashFuncID = (i-1) / 2; + assert(g_hashesToBench[hashFuncID].name != NULL); + if (g_testIDs[i] == 0) continue; + /* aligned */ + if ((i % 2) == 1) { + XSUM_benchHash(g_hashesToBench[hashFuncID].func, g_hashesToBench[hashFuncID].name, i, buffer, bufferSize); + } + /* unaligned */ + if ((i % 2) == 0) { + /* Append "unaligned". */ + char* const hashNameBuf = XSUM_strcatDup(g_hashesToBench[hashFuncID].name, " unaligned"); + assert(hashNameBuf != NULL); + XSUM_benchHash(g_hashesToBench[hashFuncID].func, hashNameBuf, i, ((const char*)buffer)+3, bufferSize); + free(hashNameBuf); + } + } } +} + +static size_t XSUM_selectBenchedSize(const char* fileName) +{ + XSUM_U64 const inFileSize = XSUM_getFileSize(fileName); + size_t benchedSize = (size_t) XSUM_findMaxMem(inFileSize); + if ((XSUM_U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize; + if (benchedSize < inFileSize) { + XSUM_log("Not enough memory for '%s' full size; testing %i MB only...\n", fileName, (int)(benchedSize>>20)); + } + return benchedSize; +} + + +int XSUM_benchFiles(const char* fileNamesTable[], int nbFiles) +{ + int fileIdx; + for (fileIdx=0; fileIdx 10 KB) { + XSUM_logVerbose(1, "%u KB", (unsigned)(keySize >> 10)); + } else { + XSUM_logVerbose(1, "%u bytes", (unsigned)keySize); + } + XSUM_logVerbose(1, "... \n"); + + XSUM_benchMem(alignedBuffer, keySize); + free(buffer); + } + return 0; +} diff --git a/deps/xxHash/cli/xsum_bench.h b/deps/xxHash/cli/xsum_bench.h new file mode 100644 index 0000000000..6faaec8c98 --- /dev/null +++ b/deps/xxHash/cli/xsum_bench.h @@ -0,0 +1,51 @@ +/* + * xsum_bench - Benchmark functions for xxhsum + * Copyright (C) 2013-2021 Yann Collet + * + * GPL v2 License + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + * + * You can contact the author at: + * - xxHash homepage: https://www.xxhash.com + * - xxHash source repository: https://github.com/Cyan4973/xxHash + */ + +#ifndef XSUM_BENCH_H +#define XSUM_BENCH_H + +#include /* size_t */ + +#define NBLOOPS_DEFAULT 3 /* Default number of benchmark iterations */ + +extern int const g_nbTestFunctions; +extern char g_testIDs[]; /* size : g_nbTestFunctions */ +extern const char k_testIDs_default[]; +extern int g_nbIterations; + +int XSUM_benchInternal(size_t keySize); +int XSUM_benchFiles(const char* fileNamesTable[], int nbFiles); + + +#ifdef __cplusplus +extern "C" { +#endif + + +#ifdef __cplusplus +} +#endif + +#endif /* XSUM_BENCH_H */ diff --git a/deps/xxHash/cli/xsum_config.h b/deps/xxHash/cli/xsum_config.h index 9222144d0e..eec5528db5 100644 --- a/deps/xxHash/cli/xsum_config.h +++ b/deps/xxHash/cli/xsum_config.h @@ -1,6 +1,6 @@ /* * xxhsum - Command line interface for xxhash algorithms - * Copyright (C) 2013-2020 Yann Collet + * Copyright (C) 2013-2021 Yann Collet * * GPL v2 License * @@ -202,4 +202,13 @@ typedef unsigned long long XSUM_U64; #endif /* not C++/C99 */ +/* *************************** + * Common constants + * ***************************/ + +#define KB *( 1<<10) +#define MB *( 1<<20) +#define GB *(1U<<30) + + #endif /* XSUM_CONFIG_H */ diff --git a/deps/xxHash/cli/xsum_os_specific.c b/deps/xxHash/cli/xsum_os_specific.c index 8f48ce0762..4f6f00fb58 100644 --- a/deps/xxHash/cli/xsum_os_specific.c +++ b/deps/xxHash/cli/xsum_os_specific.c @@ -1,6 +1,6 @@ /* * xxhsum - Command line interface for xxhash algorithms - * Copyright (C) 2013-2020 Yann Collet + * Copyright (C) 2013-2021 Yann Collet * * GPL v2 License * @@ -23,12 +23,7 @@ * - xxHash source repository: https://github.com/Cyan4973/xxHash */ -#include "xsum_config.h" -#include "xsum_os_specific.h" -#include -#include -#include -#include /* struct stat / __wstat64 */ +#include "xsum_os_specific.h" /* XSUM_API */ #include /* stat() / _stat64() */ /* @@ -47,7 +42,8 @@ #if (defined(__linux__) && (XSUM_PLATFORM_POSIX_VERSION >= 1)) \ || (XSUM_PLATFORM_POSIX_VERSION >= 200112L) \ || defined(__DJGPP__) \ - || defined(__MSYS__) + || defined(__MSYS__) \ + || defined(__HAIKU__) # include /* isatty */ # define XSUM_IS_CONSOLE(stdStream) isatty(fileno(stdStream)) #elif defined(MSDOS) || defined(OS2) @@ -112,7 +108,7 @@ static int XSUM_stat(const char* infilename, XSUM_stat_t* statbuf) } #ifndef XSUM_NO_MAIN -int main(int argc, char* argv[]) +int main(int argc, const char* argv[]) { return XSUM_main(argc, argv); } @@ -383,7 +379,7 @@ static int XSUM_wmain(int argc, wchar_t* utf16_argv[]) setvbuf(stderr, NULL, _IONBF, 0); /* Call our real main function */ - ret = XSUM_main(argc, utf8_argv); + ret = XSUM_main(argc, (void*)utf8_argv); /* Cleanup */ XSUM_freeArgv(argc, utf8_argv); @@ -439,7 +435,7 @@ int __cdecl __wgetmainargs( _startupinfo* StartInfo ); -int main(int ansi_argc, char* ansi_argv[]) +int main(int ansi_argc, const char* ansi_argv[]) { int utf16_argc; wchar_t** utf16_argv; diff --git a/deps/xxHash/cli/xsum_os_specific.h b/deps/xxHash/cli/xsum_os_specific.h index b3562b260d..e1e080a43f 100644 --- a/deps/xxHash/cli/xsum_os_specific.h +++ b/deps/xxHash/cli/xsum_os_specific.h @@ -1,6 +1,6 @@ /* * xxhsum - Command line interface for xxhash algorithms - * Copyright (C) 2013-2020 Yann Collet + * Copyright (C) 2013-2021 Yann Collet * * GPL v2 License * @@ -39,7 +39,7 @@ extern "C" { * * Functions like main(), but is passed UTF-8 arguments even on Windows. */ -XSUM_API int XSUM_main(int argc, char* argv[]); +XSUM_API int XSUM_main(int argc, const char* argv[]); /* * Returns whether stream is a console. diff --git a/deps/xxHash/cli/xsum_output.c b/deps/xxHash/cli/xsum_output.c index a4d741154d..ee7002bd42 100644 --- a/deps/xxHash/cli/xsum_output.c +++ b/deps/xxHash/cli/xsum_output.c @@ -1,6 +1,6 @@ /* * xxhsum - Command line interface for xxhash algorithms - * Copyright (C) 2013-2020 Yann Collet + * Copyright (C) 2013-2021 Yann Collet * * GPL v2 License * @@ -24,8 +24,7 @@ */ #include "xsum_output.h" -#include "xsum_os_specific.h" -#include +#include "xsum_os_specific.h" /* XSUM_API */ int XSUM_logLevel = 2; diff --git a/deps/xxHash/cli/xsum_output.h b/deps/xxHash/cli/xsum_output.h index 8a02c1b777..9a7926dfcd 100644 --- a/deps/xxHash/cli/xsum_output.h +++ b/deps/xxHash/cli/xsum_output.h @@ -1,6 +1,6 @@ /* * xxhsum - Command line interface for xxhash algorithms - * Copyright (C) 2013-2020 Yann Collet + * Copyright (C) 2013-2021 Yann Collet * * GPL v2 License * diff --git a/deps/xxHash/cli/xsum_sanity_check.c b/deps/xxHash/cli/xsum_sanity_check.c index 347d1db58e..4707735eca 100644 --- a/deps/xxHash/cli/xsum_sanity_check.c +++ b/deps/xxHash/cli/xsum_sanity_check.c @@ -1,6 +1,6 @@ /* * xxhsum - Command line interface for xxhash algorithms - * Copyright (C) 2013-2020 Yann Collet + * Copyright (C) 2013-2021 Yann Collet * * GPL v2 License * @@ -23,17 +23,17 @@ * - xxHash source repository: https://github.com/Cyan4973/xxHash */ -#include "xsum_config.h" #include "xsum_sanity_check.h" -#include "xsum_output.h" -#include -#include -#include +#include "xsum_output.h" /* XSUM_log */ #ifndef XXH_STATIC_LINKING_ONLY # define XXH_STATIC_LINKING_ONLY #endif #include "../xxhash.h" +#include /* exit */ +#include +#include /* memcmp */ + /* use #define to make them constant, required for initialization */ #define PRIME32 2654435761U #define PRIME64 11400714785074694797ULL @@ -60,7 +60,7 @@ XSUM_API void XSUM_fillTestBuffer(XSUM_U8* buffer, size_t len) /* ************************************************ * Self-test: - * ensure results consistency accross platforms + * ensure results consistency across platforms *********************************************** */ #if XSUM_NO_TESTS XSUM_API void XSUM_sanityCheck(void) @@ -90,9 +90,10 @@ typedef struct { XXH128_hash_t Nresult; } XSUM_testdata128_t; -#define SECRET_SAMPLE_NBBYTES 4 +#define SECRET_SAMPLE_NBBYTES 5 typedef struct { - XSUM_U32 len; + XSUM_U32 seedLen; + XSUM_U32 secretLen; XSUM_U8 byte[SECRET_SAMPLE_NBBYTES]; } XSUM_testdata_sample_t; @@ -213,11 +214,12 @@ static const XSUM_testdata128_t XSUM_XXH128_withSecret_testdata[] = { { 12, 0, { 0xAF82F6EBA263D7D8ULL, 0x90A3C2D839F57D0FULL } } /* 9 - 16 */ }; +#define SECRET_SIZE_MAX 9867 static const XSUM_testdata_sample_t XSUM_XXH3_generateSecret_testdata[] = { - { 0, { 0xB8, 0x26, 0x83, 0x7E } }, - { 1, { 0xA6, 0x16, 0x06, 0x7B } }, - { XXH3_SECRET_SIZE_MIN - 1, { 0xDA, 0x2A, 0x12, 0x11 } }, - { XXH3_SECRET_DEFAULT_SIZE + 500, { 0x7E, 0x48, 0x0C, 0xA7 } } + { 0, 192, { 0xE7, 0x8C, 0x77, 0x77, 0x00 } }, + { 1, 240, { 0x2B, 0x3E, 0xDE, 0xC1, 0x00 } }, + { XXH3_SECRET_SIZE_MIN - 1, 277, { 0xE8, 0x39, 0x6C, 0xCC, 0x7B } }, + { XXH3_SECRET_DEFAULT_SIZE + 500, SECRET_SIZE_MAX, { 0xD6, 0x1C, 0x41, 0x17, 0xB3 } } }; static void XSUM_checkResult32(XXH32_hash_t r1, XXH32_hash_t r2) @@ -378,6 +380,16 @@ static void XSUM_testXXH3(const void* data, const XSUM_testdata64_t* testData) XSUM_checkResult64(Dresult, Nresult); } + /* check that the combination of + * XXH3_generateSecret_fromSeed() and XXH3_64bits_withSecretandSeed() + * results in exactly the same hash generation as XXH3_64bits_withSeed() */ + { char secretBuffer[XXH3_SECRET_DEFAULT_SIZE+1]; + char* const secret = secretBuffer + 1; /* intentional unalignment */ + XXH3_generateSecret_fromSeed(secret, seed); + { XSUM_U64 const Dresult = XXH3_64bits_withSecretandSeed(data, len, secret, XXH3_SECRET_DEFAULT_SIZE, seed); + XSUM_checkResult64(Dresult, Nresult); + } } + /* streaming API test */ { XXH3_state_t* const state = XXH3_createState(); assert(state != NULL); @@ -398,10 +410,29 @@ static void XSUM_testXXH3(const void* data, const XSUM_testdata64_t* testData) (void)XXH3_64bits_update(state, ((const char*)data)+pos, 1); XSUM_checkResult64(XXH3_64bits_digest(state), Nresult); } + + /* check that streaming with a combination of + * XXH3_generateSecret_fromSeed() and XXH3_64bits_reset_withSecretandSeed() + * results in exactly the same hash generation as XXH3_64bits_reset_withSeed() */ + { char secretBuffer[XXH3_SECRET_DEFAULT_SIZE+1]; + char* const secret = secretBuffer + 1; /* intentional unalignment */ + XXH3_generateSecret_fromSeed(secret, seed); + /* single ingestion */ + (void)XXH3_64bits_reset_withSecretandSeed(state, secret, XXH3_SECRET_DEFAULT_SIZE, seed); + (void)XXH3_64bits_update(state, data, len); + XSUM_checkResult64(XXH3_64bits_digest(state), Nresult); + } + XXH3_freeState(state); } + } + +#ifndef XXH3_MIDSIZE_MAX +# define XXH3_MIDSIZE_MAX 240 +#endif + static void XSUM_testXXH3_withSecret(const void* data, const void* secret, size_t secretSize, const XSUM_testdata64_t* testData) { @@ -417,6 +448,13 @@ static void XSUM_testXXH3_withSecret(const void* data, const void* secret, XSUM_checkResult64(Dresult, Nresult); } + /* check that XXH3_64bits_withSecretandSeed() + * results in exactly the same return value as XXH3_64bits_withSecret() */ + if (len > XXH3_MIDSIZE_MAX) + { XSUM_U64 const Dresult = XXH3_64bits_withSecretandSeed(data, len, secret, secretSize, 0); + XSUM_checkResult64(Dresult, Nresult); + } + /* streaming API test */ { XXH3_state_t *state = XXH3_createState(); assert(state != NULL); @@ -436,6 +474,16 @@ static void XSUM_testXXH3_withSecret(const void* data, const void* secret, (void)XXH3_64bits_update(state, ((const char*)data)+pos, 1); XSUM_checkResult64(XXH3_64bits_digest(state), Nresult); } + + /* check that XXH3_64bits_reset_withSecretandSeed() + * results in exactly the same return value as XXH3_64bits_reset_withSecret() */ + if (len > XXH3_MIDSIZE_MAX) { + /* single ingestion */ + (void)XXH3_64bits_reset_withSecretandSeed(state, secret, secretSize, 0); + (void)XXH3_64bits_update(state, data, len); + XSUM_checkResult64(XXH3_64bits_digest(state), Nresult); + } + XXH3_freeState(state); } } @@ -466,6 +514,16 @@ static void XSUM_testXXH128(const void* data, const XSUM_testdata128_t* testData XSUM_checkResult128(Dresult, Nresult); } + /* check that the combination of + * XXH3_generateSecret_fromSeed() and XXH3_128bits_withSecretandSeed() + * results in exactly the same hash generation as XXH3_64bits_withSeed() */ + { char secretBuffer[XXH3_SECRET_DEFAULT_SIZE+1]; + char* const secret = secretBuffer + 1; /* intentional unalignment */ + XXH3_generateSecret_fromSeed(secret, seed); + { XXH128_hash_t const Dresult = XXH3_128bits_withSecretandSeed(data, len, secret, XXH3_SECRET_DEFAULT_SIZE, seed); + XSUM_checkResult128(Dresult, Nresult); + } } + /* streaming API test */ { XXH3_state_t *state = XXH3_createState(); assert(state != NULL); @@ -487,6 +545,19 @@ static void XSUM_testXXH128(const void* data, const XSUM_testdata128_t* testData (void)XXH3_128bits_update(state, ((const char*)data)+pos, 1); XSUM_checkResult128(XXH3_128bits_digest(state), Nresult); } + + /* check that streaming with a combination of + * XXH3_generateSecret_fromSeed() and XXH3_128bits_reset_withSecretandSeed() + * results in exactly the same hash generation as XXH3_128bits_reset_withSeed() */ + { char secretBuffer[XXH3_SECRET_DEFAULT_SIZE+1]; + char* const secret = secretBuffer + 1; /* intentional unalignment */ + XXH3_generateSecret_fromSeed(secret, seed); + /* single ingestion */ + (void)XXH3_128bits_reset_withSecretandSeed(state, secret, XXH3_SECRET_DEFAULT_SIZE, seed); + (void)XXH3_128bits_update(state, data, len); + XSUM_checkResult128(XXH3_128bits_digest(state), Nresult); + } + XXH3_freeState(state); } } @@ -504,6 +575,13 @@ static void XSUM_testXXH128_withSecret(const void* data, const void* secret, siz XSUM_checkResult128(Dresult, Nresult); } + /* check that XXH3_128bits_withSecretandSeed() + * results in exactly the same return value as XXH3_128bits_withSecret() */ + if (len > XXH3_MIDSIZE_MAX) + { XXH128_hash_t const Dresult = XXH3_128bits_withSecretandSeed(data, len, secret, secretSize, 0); + XSUM_checkResult128(Dresult, Nresult); + } + /* streaming API test */ { XXH3_state_t* const state = XXH3_createState(); assert(state != NULL); @@ -523,6 +601,16 @@ static void XSUM_testXXH128_withSecret(const void* data, const void* secret, siz (void)XXH3_128bits_update(state, ((const char*)data)+pos, 1); XSUM_checkResult128(XXH3_128bits_digest(state), Nresult); } + + /* check that XXH3_128bits_reset_withSecretandSeed() + * results in exactly the same return value as XXH3_128bits_reset_withSecret() */ + if (len > XXH3_MIDSIZE_MAX) { + /* single ingestion */ + (void)XXH3_128bits_reset_withSecretandSeed(state, secret, secretSize, 0); + (void)XXH3_128bits_update(state, data, len); + XSUM_checkResult128(XXH3_128bits_digest(state), Nresult); + } + XXH3_freeState(state); } } @@ -530,20 +618,21 @@ static void XSUM_testXXH128_withSecret(const void* data, const void* secret, siz static void XSUM_testSecretGenerator(const void* customSeed, const XSUM_testdata_sample_t* testData) { static int nbTests = 1; - const int sampleIndex[SECRET_SAMPLE_NBBYTES] = { 0, 62, 131, 191}; - XSUM_U8 secretBuffer[XXH3_SECRET_DEFAULT_SIZE] = {0}; + const int sampleIndex[SECRET_SAMPLE_NBBYTES] = { 0, 62, 131, 191, 241 }; /* position of sampled bytes */ + XSUM_U8 secretBuffer[SECRET_SIZE_MAX] = {0}; XSUM_U8 samples[SECRET_SAMPLE_NBBYTES]; int i; - XXH3_generateSecret(secretBuffer, customSeed, testData->len); + assert(testData->secretLen <= SECRET_SIZE_MAX); + XXH3_generateSecret(secretBuffer, testData->secretLen, customSeed, testData->seedLen); for (i=0; ibyte, sizeof(testData->byte))) { XSUM_log("\rError: Secret generation test %i: Internal sanity check failed. \n", nbTests); - XSUM_log("\rGot { 0x%02X, 0x%02X, 0x%02X, 0x%02X }, expected { 0x%02X, 0x%02X, 0x%02X, 0x%02X } \n", - samples[0], samples[1], samples[2], samples[3], - testData->byte[0], testData->byte[1], testData->byte[2], testData->byte[3] ); + XSUM_log("\rGot { 0x%02X, 0x%02X, 0x%02X, 0x%02X, 0x%02X }, expected { 0x%02X, 0x%02X, 0x%02X, 0x%02X, 0x%02X } \n", + samples[0], samples[1], samples[2], samples[3], samples[4], + testData->byte[0], testData->byte[1], testData->byte[2], testData->byte[3], testData->byte[4] ); exit(1); } nbTests++; @@ -592,6 +681,7 @@ XSUM_API void XSUM_sanityCheck(void) } /* secret generator */ for (i = 0; i < (sizeof(XSUM_XXH3_generateSecret_testdata)/sizeof(XSUM_XXH3_generateSecret_testdata[0])); i++) { + assert(XSUM_XXH3_generateSecret_testdata[i].seedLen <= SANITY_BUFFER_SIZE); XSUM_testSecretGenerator(sanityBuffer, &XSUM_XXH3_generateSecret_testdata[i]); } diff --git a/deps/xxHash/cli/xsum_sanity_check.h b/deps/xxHash/cli/xsum_sanity_check.h index 4e3bc0f69d..af766dfa4f 100644 --- a/deps/xxHash/cli/xsum_sanity_check.h +++ b/deps/xxHash/cli/xsum_sanity_check.h @@ -1,6 +1,6 @@ /* * xxhsum - Command line interface for xxhash algorithms - * Copyright (C) 2013-2020 Yann Collet + * Copyright (C) 2013-2021 Yann Collet * * GPL v2 License * diff --git a/deps/xxHash/cli/xxhsum.1 b/deps/xxHash/cli/xxhsum.1 index dd17108f10..715fc4be9b 100644 --- a/deps/xxHash/cli/xxhsum.1 +++ b/deps/xxHash/cli/xxhsum.1 @@ -1,165 +1,106 @@ -. -.TH "XXHSUM" "1" "July 2020" "xxhsum 0.7.4" "User Commands" -. +.TH "XXHSUM" "1" "December 2021" "xxhsum 0.8.1" "User Commands" .SH "NAME" \fBxxhsum\fR \- print or check xxHash non\-cryptographic checksums -. .SH "SYNOPSIS" -\fBxxhsum [